From e7ab4671a36222f0c444c3f7845c0950589c0fba Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Wed, 13 Jan 2021 15:33:44 -0600 Subject: [PATCH 1/2] Add third-party REST API - apache - nginx - zeek - aws/cloudtrail --- .../access/agent/stream/httpjson.yml.hbs | 87 ++++++ .../elasticsearch/ingest_pipeline/default.yml | 4 + .../apache/data_stream/access/manifest.yml | 26 ++ .../error/agent/stream/httpjson.yml.hbs | 87 ++++++ .../apache/data_stream/error/manifest.yml | 26 ++ packages/apache/manifest.yml | 53 +++- .../cloudtrail/agent/stream/httpjson.yml.hbs | 80 ++++++ .../aws/data_stream/cloudtrail/manifest.yml | 26 ++ packages/aws/manifest.yml | 53 +++- .../access/agent/stream/httpjson.yml.hbs | 86 ++++++ .../elasticsearch/ingest_pipeline/default.yml | 4 + .../nginx/data_stream/access/manifest.yml | 26 ++ .../error/agent/stream/httpjson.yml.hbs | 86 ++++++ .../elasticsearch/ingest_pipeline/default.yml | 4 + packages/nginx/data_stream/error/manifest.yml | 26 ++ packages/nginx/manifest.yml | 53 +++- .../agent/stream/httpjson.yml.hbs | 87 ++++++ .../data_stream/capture_loss/manifest.yml | 26 ++ .../connection/agent/stream/httpjson.yml.hbs | 153 ++++++++++ .../zeek/data_stream/connection/manifest.yml | 32 ++- .../dce_rpc/agent/stream/httpjson.yml.hbs | 121 ++++++++ .../zeek/data_stream/dce_rpc/manifest.yml | 26 ++ .../dhcp/agent/stream/httpjson.yml.hbs | 167 +++++++++++ packages/zeek/data_stream/dhcp/manifest.yml | 26 ++ .../dnp3/agent/stream/httpjson.yml.hbs | 128 +++++++++ packages/zeek/data_stream/dnp3/manifest.yml | 26 ++ .../dns/agent/stream/httpjson.yml.hbs | 270 ++++++++++++++++++ packages/zeek/data_stream/dns/manifest.yml | 26 ++ .../dpd/agent/stream/httpjson.yml.hbs | 120 ++++++++ packages/zeek/data_stream/dpd/manifest.yml | 26 ++ .../files/agent/stream/httpjson.yml.hbs | 111 +++++++ packages/zeek/data_stream/files/manifest.yml | 26 ++ .../ftp/agent/stream/httpjson.yml.hbs | 141 +++++++++ packages/zeek/data_stream/ftp/manifest.yml | 26 ++ .../http/agent/stream/httpjson.yml.hbs | 143 ++++++++++ packages/zeek/data_stream/http/manifest.yml | 26 ++ .../intel/agent/stream/httpjson.yml.hbs | 136 +++++++++ packages/zeek/data_stream/intel/manifest.yml | 26 ++ .../irc/agent/stream/httpjson.yml.hbs | 132 +++++++++ packages/zeek/data_stream/irc/manifest.yml | 26 ++ .../kerberos/agent/stream/httpjson.yml.hbs | 155 ++++++++++ .../zeek/data_stream/kerberos/manifest.yml | 26 ++ .../modbus/agent/stream/httpjson.yml.hbs | 135 +++++++++ packages/zeek/data_stream/modbus/manifest.yml | 26 ++ .../mysql/agent/stream/httpjson.yml.hbs | 135 +++++++++ packages/zeek/data_stream/mysql/manifest.yml | 26 ++ .../notice/agent/stream/httpjson.yml.hbs | 152 ++++++++++ packages/zeek/data_stream/notice/manifest.yml | 26 ++ .../ntlm/agent/stream/httpjson.yml.hbs | 145 ++++++++++ packages/zeek/data_stream/ntlm/manifest.yml | 26 ++ .../ocsp/agent/stream/httpjson.yml.hbs | 120 ++++++++ packages/zeek/data_stream/ocsp/manifest.yml | 26 ++ .../pe/agent/stream/httpjson.yml.hbs | 101 +++++++ packages/zeek/data_stream/pe/manifest.yml | 26 ++ .../radius/agent/stream/httpjson.yml.hbs | 121 ++++++++ packages/zeek/data_stream/radius/manifest.yml | 26 ++ .../rdp/agent/stream/httpjson.yml.hbs | 140 +++++++++ packages/zeek/data_stream/rdp/manifest.yml | 26 ++ .../rfb/agent/stream/httpjson.yml.hbs | 130 +++++++++ packages/zeek/data_stream/rfb/manifest.yml | 26 ++ .../sip/agent/stream/httpjson.yml.hbs | 146 ++++++++++ packages/zeek/data_stream/sip/manifest.yml | 26 ++ .../smb_cmd/agent/stream/httpjson.yml.hbs | 154 ++++++++++ .../zeek/data_stream/smb_cmd/manifest.yml | 26 ++ .../smb_files/agent/stream/httpjson.yml.hbs | 124 ++++++++ .../zeek/data_stream/smb_files/manifest.yml | 26 ++ .../smb_mapping/agent/stream/httpjson.yml.hbs | 120 ++++++++ .../zeek/data_stream/smb_mapping/manifest.yml | 26 ++ .../smtp/agent/stream/httpjson.yml.hbs | 127 ++++++++ packages/zeek/data_stream/smtp/manifest.yml | 26 ++ .../snmp/agent/stream/httpjson.yml.hbs | 128 +++++++++ packages/zeek/data_stream/snmp/manifest.yml | 26 ++ .../socks/agent/stream/httpjson.yml.hbs | 127 ++++++++ packages/zeek/data_stream/socks/manifest.yml | 26 ++ .../ssh/agent/stream/httpjson.yml.hbs | 132 +++++++++ packages/zeek/data_stream/ssh/manifest.yml | 26 ++ .../ssl/agent/stream/httpjson.yml.hbs | 135 +++++++++ packages/zeek/data_stream/ssl/manifest.yml | 26 ++ .../stats/agent/stream/httpjson.yml.hbs | 141 +++++++++ packages/zeek/data_stream/stats/manifest.yml | 26 ++ .../syslog/agent/stream/httpjson.yml.hbs | 118 ++++++++ packages/zeek/data_stream/syslog/manifest.yml | 26 ++ .../traceroute/agent/stream/httpjson.yml.hbs | 111 +++++++ .../zeek/data_stream/traceroute/manifest.yml | 26 ++ .../tunnel/agent/stream/httpjson.yml.hbs | 119 ++++++++ packages/zeek/data_stream/tunnel/manifest.yml | 26 ++ .../weird/agent/stream/httpjson.yml.hbs | 119 ++++++++ packages/zeek/data_stream/weird/manifest.yml | 26 ++ .../x509/agent/stream/httpjson.yml.hbs | 123 ++++++++ packages/zeek/data_stream/x509/manifest.yml | 26 ++ packages/zeek/manifest.yml | 53 +++- 91 files changed, 6704 insertions(+), 11 deletions(-) create mode 100644 packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs create mode 100644 packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs create mode 100644 packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs create mode 100644 packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs create mode 100644 packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs create mode 100644 packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs diff --git a/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs b/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..c0c14d398d7 --- /dev/null +++ b/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs @@ -0,0 +1,87 @@ +config_version: "2" +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - drop_fields: + fields: message + - rename: + fields: + - from: json.result._raw + to: message + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + ignore_missing: true + fail_on_error: false + - drop_fields: + fields: json + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 9e0d5272bed..df8321d9655 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -40,6 +40,10 @@ processors: ignore_missing: true patterns: - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ +- remove: + field: event.created + ignore_missing: true + ignore_failure: true - rename: field: '@timestamp' target_field: event.created diff --git a/packages/apache/data_stream/access/manifest.yml b/packages/apache/data_stream/access/manifest.yml index a339bc28d71..a618c42e449 100644 --- a/packages/apache/data_stream/access/manifest.yml +++ b/packages/apache/data_stream/access/manifest.yml @@ -17,3 +17,29 @@ streams: template_path: log.yml.hbs title: Apache access logs description: Collect Apache access logs + - input: httpjson + title: Apache access logs via Splunk Enterprise REST API + description: Collect apache access logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"access*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs b/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..021204e69b1 --- /dev/null +++ b/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs @@ -0,0 +1,87 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - drop_fields: + fields: message + - rename: + fields: + - from: json.result._raw + to: message + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + ignore_missing: true + fail_on_error: false + - drop_fields: + fields: json + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 diff --git a/packages/apache/data_stream/error/manifest.yml b/packages/apache/data_stream/error/manifest.yml index a50c1e4b601..f9338412651 100644 --- a/packages/apache/data_stream/error/manifest.yml +++ b/packages/apache/data_stream/error/manifest.yml @@ -16,3 +16,29 @@ streams: template_path: log.yml.hbs title: Apache error logs description: Collect Apache error logs + - input: httpjson + title: Apache error logs via Splunk Enterprise REST API + description: Collect apache error logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: search sourcetype=apache:error OR sourcetype=apache_error + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml index 91087abff14..20a324fa20b 100644 --- a/packages/apache/manifest.yml +++ b/packages/apache/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: apache title: Apache -version: 0.3.4 +version: 0.4.0 license: basic description: Apache Integration type: integration @@ -9,7 +9,7 @@ categories: - web release: experimental conditions: - kibana.version: '^7.11.0' + kibana.version: '^7.12.0' screenshots: - src: /img/apache-metrics-overview.png title: Apache metrics overview @@ -32,6 +32,55 @@ policy_templates: - type: logfile title: Collect logs from Apache instances description: Collecting Apache access and error logs + - type: httpjson + title: Collect Apache logs from third-party REST API + description: Collecting Apache logs via third-party REST API + vars: + - name: server + type: text + title: Address of Splunk Enterprise Server + description: hostname or IP + show_user: true + required: true + default: server.example.com + - name: port + type: text + title: Port number of Splunk Enterprise REST API + show_user: true + required: true + default: 8089 + - name: username + type: text + title: Splunk REST API Username + show_user: true + required: true + - name: password + type: password + title: Splunk REST API Password + required: true + show_user: true + - name: ssl.enabled + type: bool + title: SSL enabled + multi: false + required: false + show_user: false + default: true + - name: ssl.verification_mode + type: text + title: Mode of server verification + description: "valid values: none, strict, certificate or full" + multi: false + required: false + show_user: false + default: full + - name: ssl.certificate_authorities + type: text + title: List of root certificates for TLS server verification + description: PEM encoded + multi: true + required: false + show_user: false - type: apache/metrics title: Collect metrics from Apache instances description: Collecting Apache status metrics diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..9de7d5a067e --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs @@ -0,0 +1,80 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + target_field: "@metadata._id" + - drop_fields: + fields: ["message"] + - rename: + fields: + - from: json.result._raw + to: message + - drop_fields: + fields: ["json"] + ignore_missing: true + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index 94b8c1d0e37..d8514fe7399 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -22,3 +22,29 @@ streams: required: false show_user: false description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - input: httpjson + title: AWS CloudTrail logs via Splunk Enterprise REST API + description: Collect AWS CloudTrail logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=aws:cloudtrail" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 42d4b0afd6c..1f10d7ec5f7 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 0.4.1 +version: 0.5.0 license: basic description: AWS Integration type: integration @@ -12,7 +12,7 @@ categories: - security release: beta conditions: - kibana.version: '^7.9.0' + kibana.version: '^7.12.0' screenshots: - src: /img/filebeat-aws-cloudtrail.png title: filebeat aws cloudtrail @@ -239,5 +239,54 @@ policy_templates: show_user: false default: "amazonaws.com" description: URL of the entry point for an AWS web service. + - type: httpjson + title: Collect AWS logs from third-party REST API + description: "Collects AWS logs from third-party REST API" + vars: + - name: server + type: text + title: Address of Splunk Enterprise Server + description: hostname or IP + show_user: true + required: true + default: server.example.com + - name: port + type: text + title: Port number of Splunk Enterprise REST API + show_user: true + required: true + default: 8089 + - name: username + type: text + title: Splunk REST API Username + show_user: true + required: true + - name: password + type: password + title: Splunk REST API Password + required: true + show_user: true + - name: ssl.enabled + type: bool + title: SSL enabled + multi: false + required: false + show_user: false + default: true + - name: ssl.verification_mode + type: text + title: Mode of server verification + description: "valid values: none, strict, certificate or full" + multi: false + required: false + show_user: false + default: full + - name: ssl.certificate_authorities + type: text + title: List of root certificates for TLS server verification + description: PEM encoded + multi: true + required: false + show_user: false owner: github: elastic/integrations diff --git a/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs b/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..7bcbe024e21 --- /dev/null +++ b/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs @@ -0,0 +1,86 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +processors: + - decode_json_fields: + fields: message + target: json + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - drop_fields: + fields: message + - rename: + fields: + - from: json.result._raw + to: message + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + ignore_missing: true + fail_on_error: false + - drop_fields: + fields: json + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 diff --git a/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 9de5d5e7c4b..2ba557b4c09 100644 --- a/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -93,6 +93,10 @@ processors: ignore_failure: true - remove: field: message +- remove: + field: event.created + ignore_missing: true + ignore_failure: true - rename: field: '@timestamp' target_field: event.created diff --git a/packages/nginx/data_stream/access/manifest.yml b/packages/nginx/data_stream/access/manifest.yml index 410f19978ae..8c8aaf40e6c 100644 --- a/packages/nginx/data_stream/access/manifest.yml +++ b/packages/nginx/data_stream/access/manifest.yml @@ -14,3 +14,29 @@ streams: - /var/log/nginx/access.log* title: Nginx access logs description: Collect Nginx access logs + - input: httpjson + title: Nginx access logs via Splunk Enterprise REST API + description: Collect Nginx access logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: search sourcetype=nginx:plus:access + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs b/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..7bcbe024e21 --- /dev/null +++ b/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs @@ -0,0 +1,86 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +processors: + - decode_json_fields: + fields: message + target: json + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - drop_fields: + fields: message + - rename: + fields: + - from: json.result._raw + to: message + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + ignore_missing: true + fail_on_error: false + - drop_fields: + fields: json + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 diff --git a/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml b/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml index 96b46eb9c98..112d057f0b1 100644 --- a/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml @@ -11,6 +11,10 @@ processors: (.| | )* ignore_missing: true +- remove: + field: event.created + ignore_missing: true + ignore_failure: true - rename: field: '@timestamp' target_field: event.created diff --git a/packages/nginx/data_stream/error/manifest.yml b/packages/nginx/data_stream/error/manifest.yml index 961092483b7..3982966c616 100644 --- a/packages/nginx/data_stream/error/manifest.yml +++ b/packages/nginx/data_stream/error/manifest.yml @@ -14,3 +14,29 @@ streams: - /var/log/nginx/error.log* title: Nginx error logs description: Collect Nginx error logs + - input: httpjson + title: Nginx error logs via Splunk REST API + description: Collect Nginx error logs via Splunk REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Search String + show_user: true + required: true + default: search sourcetype=nginx:plus:error + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/nginx/manifest.yml b/packages/nginx/manifest.yml index c18b75dc1e3..4a6a279a572 100644 --- a/packages/nginx/manifest.yml +++ b/packages/nginx/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: nginx title: Nginx -version: 0.3.10 +version: 0.4.0 license: basic description: Nginx Integration type: integration @@ -10,7 +10,7 @@ categories: - security release: experimental conditions: - kibana.version: '^7.11.0' + kibana.version: '^7.12.0' screenshots: - src: /img/nginx-metrics-overview.png title: Nginx metrics overview @@ -37,6 +37,55 @@ policy_templates: - type: logfile title: Collect logs from Nginx instances description: Collecting Nginx access and error logs + - type: httpjson + title: Collect Nginx logs from third-party REST API + description: Collecting Nginx logs via third-party REST API + vars: + - name: server + type: text + title: Address of Splunk Enterprise Server + description: hostname or IP + show_user: true + required: true + default: server.example.com + - name: port + type: text + title: Port number of Splunk Enterprise REST API + show_user: true + required: true + default: 8089 + - name: username + type: text + title: Splunk REST API Username + show_user: true + required: true + - name: password + type: password + title: Splunk REST API Password + required: true + show_user: true + - name: ssl.enabled + type: bool + title: SSL enabled + multi: false + required: false + show_user: false + default: true + - name: ssl.verification_mode + type: text + title: Mode of server verification + description: "valid values: none, strict, certificate or full" + multi: false + required: false + show_user: false + default: full + - name: ssl.certificate_authorities + type: text + title: List of root certificates for TLS server verification + description: PEM encoded + multi: true + required: false + show_user: false - type: nginx/metrics vars: - name: hosts diff --git a/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..747247307c9 --- /dev/null +++ b/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs @@ -0,0 +1,87 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.capture_loss + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/capture_loss/manifest.yml b/packages/zeek/data_stream/capture_loss/manifest.yml index 26ae99186e4..b0ebb638666 100644 --- a/packages/zeek/data_stream/capture_loss/manifest.yml +++ b/packages/zeek/data_stream/capture_loss/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek capture_loss.log description: Collect Zeek capture_loss logs + - input: httpjson + title: Zeek capture_loss logs via Splunk Enterprise REST API + description: Collect Zeek capture_loss logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"capture_loss-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..78a56a49d64 --- /dev/null +++ b/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs @@ -0,0 +1,153 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.connection + - rename: + fields: + - from: zeek.connection.duration + to: temp.duration + - from: zeek.connection.id.orig_h + to: source.address + - from: zeek.connection.id.orig_p + to: source.port + - from: zeek.connection.id.resp_h + to: destination.address + - from: zeek.connection.id.resp_p + to: destination.port + - from: zeek.connection.proto + to: network.transport + - from: zeek.connection.service + to: network.protocol + - from: zeek.connection.uid + to: zeek.session_id + - from: zeek.connection.orig_ip_bytes + to: source.bytes + - from: zeek.connection.resp_ip_bytes + to: destination.bytes + - from: zeek.connection.orig_pkts + to: source.packets + - from: zeek.connection.resp_pkts + to: destination.packets + - from: zeek.connection.conn_state + to: zeek.connection.state + - from: zeek.connection.orig_l2_addr + to: source.mac + - from: zeek.connection.resp_l2_addr + to: destination.mac + ignore_missing: true + fail_on_error: false + - rename: + when.equals.network.transport: icmp + fields: + - from: source.port + to: zeek.connection.icmp.type + - from: destination.port + to: zeek.connection.icmp.code + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + - if: + equals.network.transport: icmp + then: + community_id: + fields: + icmp_type: zeek.connection.icmp.type + icmp_code: zeek.connection.icmp.code + else: + community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["zeek.connection.orig_bytes","zeek.connection.resp_bytes","zeek.connection.tunnel_parents","json","message"] + ignore_missing: true + diff --git a/packages/zeek/data_stream/connection/manifest.yml b/packages/zeek/data_stream/connection/manifest.yml index 6a0d3a757e5..c432b4ddf88 100644 --- a/packages/zeek/data_stream/connection/manifest.yml +++ b/packages/zeek/data_stream/connection/manifest.yml @@ -3,6 +3,9 @@ title: Zeek connection logs release: experimental streams: - input: logfile + template_path: log.yml.hbs + title: Zeek conn.log + description: Collect Zeek connection logs vars: - name: filenames type: text @@ -20,6 +23,29 @@ streams: show_user: true default: - zeek.connection - template_path: log.yml.hbs - title: Zeek conn.log - description: Collect Zeek connection logs + - input: httpjson + title: Zeek connection logs via Splunk Enterprise REST API + description: Collect Zeek connection logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"conn-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..7ed931f7add --- /dev/null +++ b/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs @@ -0,0 +1,121 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.dce_rpc + - rename: + fields: + - from: "zeek.dce_rpc.id.orig_h" + to: "source.address" + - from: "zeek.dce_rpc.id.orig_p" + to: "source.port" + - from: "zeek.dce_rpc.id.resp_h" + to: "destination.address" + - from: "zeek.dce_rpc.id.resp_p" + to: "destination.port" + - from: "zeek.dce_rpc.uid" + to: "zeek.session_id" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: dce_rpc + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/dce_rpc/manifest.yml b/packages/zeek/data_stream/dce_rpc/manifest.yml index 8abc55827a1..34eaeccdb04 100644 --- a/packages/zeek/data_stream/dce_rpc/manifest.yml +++ b/packages/zeek/data_stream/dce_rpc/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek dce_rpc.log description: Collect Zeek dce_rpc logs + - input: httpjson + title: Zeek dce_rpc logs via Splunk Enterprise REST API + description: Collect Zeek dce_rpc logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"dce_rpc-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..3fd3a2a74a1 --- /dev/null +++ b/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs @@ -0,0 +1,167 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.dhcp + - rename: + fields: + - from: "zeek.dhcp.uids" + to: "zeek.session_id" + - from: "zeek.dhcp.assigned_addr" + to: "zeek.dhcp.address.assigned" + - from: "zeek.dhcp.client_addr" + to: "zeek.dhcp.address.client" + - from: "zeek.dhcp.mac" + to: "zeek.dhcp.address.mac" + - from: "zeek.dhcp.requested_addr" + to: "zeek.dhcp.address.requested" + - from: "zeek.dhcp.server_addr" + to: "zeek.dhcp.address.server" + - from: "zeek.dhcp.host_name" + to: "zeek.dhcp.hostname" + - from: "zeek.dhcp.client_message" + to: "zeek.dhcp.msg.client" + - from: "zeek.dhcp.server_message" + to: "zeek.dhcp.msg.server" + - from: "zeek.dhcp.msg_types" + to: "zeek.dhcp.msg.types" + - from: "zeek.dhcp.msg_orig" + to: "zeek.dhcp.msg.origin" + - from: "zeek.dhcp.client_software" + to: "zeek.dhcp.software.client" + - from: "zeek.dhcp.server_software" + to: "zeek.dhcp.software.server" + - from: "zeek.dhcp.circuit_id" + to: "zeek.dhcp.id.circuit" + - from: "zeek.dhcp.agent_remote_id" + to: "zeek.dhcp.id.remote_agent" + - from: "zeek.dhcp.subscriber_id" + to: "zeek.dhcp.id.subscriber" + - from: "zeek.dhcp.client_port" + to: "source.port" + - from: "zeek.dhcp.server_port" + to: "destination.port" + ignore_missing: true + fail_on_error: false + - if: + not: + has_fields: ["source.port"] + then: + - add_fields: + target: source + fields: + port: 68 + - if: + not: + has_fields: ["destination.port"] + then: + - add_fields: + target: destination + fields: + port: 67 + - convert: + fields: + - {from: "zeek.dhcp.address.client", to: "source.address"} + - {from: "zeek.dhcp.address.client", to: "source.ip", type: "ip"} + - {from: "zeek.dhcp.address.client", to: "client.address"} + - {from: "zeek.dhcp.address.server", to: "destination.address"} + - {from: "zeek.dhcp.address.server", to: "destination.ip", type: "ip"} + - {from: "zeek.dhcp.address.server", to: "server.address"} + - {from: "zeek.dhcp.domain", to: "network.name"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: udp + network.protocol: dhcp + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/dhcp/manifest.yml b/packages/zeek/data_stream/dhcp/manifest.yml index fcdc5b42ff4..dd7c9a4145f 100644 --- a/packages/zeek/data_stream/dhcp/manifest.yml +++ b/packages/zeek/data_stream/dhcp/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek dhcp.log description: Collect Zeek dhcp logs + - input: httpjson + title: Zeek dhcp logs via Splunk Enterprise REST API + description: Collect Zeek dhcp logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"dhcp-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..a4aeea075df --- /dev/null +++ b/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs @@ -0,0 +1,128 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.dnp3 + - rename: + fields: + - from: "zeek.dnp3.id.orig_h" + to: "source.address" + - from: "zeek.dnp3.id.orig_p" + to: "source.port" + - from: "zeek.dnp3.id.resp_h" + to: "destination.address" + - from: "zeek.dnp3.id.resp_p" + to: "destination.port" + - from: "zeek.dnp3.uid" + to: "event.id" + - from: "zeek.dnp3.fc_request" + to: "zeek.dnp3.function.request" + - from: "zeek.dnp3.fc_reply" + to: "zeek.dnp3.function.reply" + - from: "zeek.dnp3.iin" + to: "zeek.dnp3.id" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: dnp3 + - drop_fields: + fields: ["json","message"] + ignore_missing: true + diff --git a/packages/zeek/data_stream/dnp3/manifest.yml b/packages/zeek/data_stream/dnp3/manifest.yml index 8f328d6f59b..e025d395171 100644 --- a/packages/zeek/data_stream/dnp3/manifest.yml +++ b/packages/zeek/data_stream/dnp3/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek dnp3.log description: Collect Zeek dnp3 logs + - input: httpjson + title: Zeek dnp3 logs via Splunk Enterprise REST API + description: Collect Zeek dnp3 logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"dnp3-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..d01b8da8f0a --- /dev/null +++ b/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs @@ -0,0 +1,270 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.dns + - registered_domain: + ignore_missing: true + fail_on_error: false + field: zeek.dns.query + target_field: dns.question.registered_domain + target_subdomain_field: dns.question.subdomain + target_etld_field: dns.question.top_level_domain + - script: + lang: javascript + id: zeek_dns_flags + source: > + var net = require("net"); + function addDnsHeaderFlags(evt) { + var flag = evt.Get("zeek.dns.AA"); + if (flag === true) { + evt.AppendTo("dns.header_flags", "AA"); + } + flag = evt.Get("zeek.dns.TC"); + if (flag === true) { + evt.AppendTo("dns.header_flags", "TC"); + } + flag = evt.Get("zeek.dns.RD"); + if (flag === true) { + evt.AppendTo("dns.header_flags", "RD"); + } + flag = evt.Get("zeek.dns.RA"); + if (flag === true) { + evt.AppendTo("dns.header_flags", "RA"); + } + } + function addDnsQuestionClass(evt) { + var qclass = evt.Get("zeek.dns.qclass"); + if (!qclass) { + return; + } + switch (qclass) { + case 1: + qclass = "IN"; + break; + case 3: + qclass = "CH"; + break; + case 4: + qclass = "HS"; + break; + case 254: + qclass = "NONE"; + break; + case 255: + qclass = "ANY"; + break; + } + evt.Put("dns.question.class", qclass); + } + function addDnsAnswers(evt) { + var answers = evt.Get("zeek.dns.answers"); + var ttls = evt.Get("zeek.dns.TTLs"); + if (!answers || !ttls || answers.length != ttls.length) { + return; + } + var resolvedIps = []; + var answersObjs = []; + for (var i = 0; i < answers.length; i++) { + var answer = answers[i]; + answersObjs.push({ + data: answer, + ttl: ttls[i], + }) + if (net.isIP(answer)) { + resolvedIps.push(answer); + } + } + evt.Put("dns.answers", answersObjs); + if (resolvedIps.length > 0) { + evt.Put("dns.resolved_ip", resolvedIps); + } + } + function setDnsType(evt) { + var response_code = evt.Get("zeek.dns.rcode_name"); + if (response_code) { + evt.Put("dns.type", "answer"); + } else { + evt.Put("dns.type", "query"); + } + } + function addEventDuration(evt) { + var rttSec = evt.Get("zeek.dns.rtt"); + if (!rttSec) { + return; + } + evt.Put("event.duration", rttSec * 1000000000); + } + function addTopLevelDomain(evt) { + var rd = evt.Get("dns.question.registered_domain"); + if (!rd) { + return; + } + var firstPeriod = rd.indexOf("."); + if (firstPeriod == -1) { + return; + } + evt.Put("dns.question.top_level_domain", rd.substr(firstPeriod + 1)); + } + function addEventOutcome(evt) { + var rcode = evt.Get("zeek.dns.rcode"); + if (rcode == null) { + return; + } + if (rcode == 0) { + evt.Put("event.outcome", "success"); + } else { + evt.Put("event.outcome", "failure"); + } + } + function addRelatedIP(evt) { + var related = []; + var src = evt.Get("zeek.dns.id.orig_h"); + if (src != null) { + related.push(src); + } + var dst = evt.Get("zeek.dns.id.resp_h"); + if (dst != null) { + related.push(dst); + } + if (related.length > 0) { + evt.Put("related.ip", related); + } + } + function process(evt) { + addDnsHeaderFlags(evt); + addDnsQuestionClass(evt); + addDnsAnswers(evt); + setDnsType(evt); + addEventDuration(evt); + addTopLevelDomain(evt); + addEventOutcome(evt); + addRelatedIP(evt); + } + - convert: + ignore_missing: true + fail_on_error: false + mode: rename + fields: + - {from: zeek.dns.id.orig_h, to: source.address} + - {from: zeek.dns.id.orig_p, to: source.port, type: long} + - {from: zeek.dns.id.resp_h, to: destination.address} + - {from: zeek.dns.id.resp_p, to: destination.port, type: long} + - {from: zeek.dns.uid, to: zeek.session_id} + - {from: zeek.dns.proto, to: network.transport} + - convert: + ignore_missing: true + fail_on_error: false + mode: copy + fields: + - {from: source.address, to: source.ip, type: ip} + - {from: destination.address, to: destination.ip, type: ip} + - {from: zeek.session_id, to: event.id} + - {from: zeek.dns.trans_id, to: dns.id, type: string} + - {from: zeek.dns.query, to: dns.question.name} + - {from: zeek.dns.qtype_name, to: dns.question.type} + - {from: zeek.dns.rcode_name, to: dns.response_code} + - convert: + ignore_missing: true + fail_on_error: false + fields: + - {from: zeek.dns.trans_id, type: string} + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - info + - protocol + - community_id: + - drop_fields: + ignore_missing: true + fields: + - zeek.dns.Z + - zeek.dns.auth + - zeek.dns.addl + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/dns/manifest.yml b/packages/zeek/data_stream/dns/manifest.yml index ca28599aea8..ae90211119a 100644 --- a/packages/zeek/data_stream/dns/manifest.yml +++ b/packages/zeek/data_stream/dns/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek dns.log description: Collect Zeek dns logs + - input: httpjson + title: Zeek dns logs via Splunk Enterprise REST API + description: Collect Zeek dns logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"dns-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..ec1a16c8615 --- /dev/null +++ b/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs @@ -0,0 +1,120 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.dpd + - rename: + fields: + - from: "zeek.dpd.id.orig_h" + to: "source.address" + - from: "zeek.dpd.id.orig_p" + to: "source.port" + - from: "zeek.dpd.id.resp_h" + to: "destination.address" + - from: "zeek.dpd.id.resp_p" + to: "destination.port" + - from: "zeek.dpd.uid" + to: "zeek.session_id" + - from: "zeek.dpd.proto" + to: "network.transport" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.session_id", to: "event.id"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/dpd/manifest.yml b/packages/zeek/data_stream/dpd/manifest.yml index ffa1ee62bc0..75f657916b6 100644 --- a/packages/zeek/data_stream/dpd/manifest.yml +++ b/packages/zeek/data_stream/dpd/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek dpd.log description: Collect Zeek dpd logs + - input: httpjson + title: Zeek dpd logs via Splunk Enterprise REST API + description: Collect Zeek dpd logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"dpd-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..6e9c749c67e --- /dev/null +++ b/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs @@ -0,0 +1,111 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.files + - rename: + fields: + - from: "zeek.files.conn_uids" + to: "zeek.files.session_ids" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.files.mime_type", to: "file.mime_type"} + - {from: "zeek.files.filename", to: "file.name"} + - {from: "zeek.files.total_bytes", to: "file.size"} + - {from: "zeek.files.md5", to: "file.hash.md5"} + - {from: "zeek.files.sha1", to: "file.hash.sha1"} + - {from: "zeek.files.sha256", to: "file.hash.sha256"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - file + type: + - info + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/files/manifest.yml b/packages/zeek/data_stream/files/manifest.yml index 50691bf72fb..f39b05d03ec 100644 --- a/packages/zeek/data_stream/files/manifest.yml +++ b/packages/zeek/data_stream/files/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek files.log description: Collect Zeek files logs + - input: httpjson + title: Zeek files logs via Splunk Enterprise REST API + description: Collect Zeek files logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"files-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..8d62b91f787 --- /dev/null +++ b/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs @@ -0,0 +1,141 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.ftp + - rename: + fields: + - from: "zeek.ftp.id.orig_h" + to: "source.address" + - from: "zeek.ftp.id.orig_p" + to: "source.port" + - from: "zeek.ftp.id.resp_h" + to: "destination.address" + - from: "zeek.ftp.id.resp_p" + to: "destination.port" + - from: "zeek.ftp.uid" + to: "zeek.session_id" + - from: "zeek.ftp.file_size" + to: "zeek.ftp.file.size" + - from: "zeek.ftp.mime_type" + to: "zeek.ftp.file.mime_type" + - from: "zeek.ftp.fuid" + to: "zeek.ftp.file.uid" + - from: "zeek.ftp.reply_code" + to: "zeek.ftp.reply.code" + - from: "zeek.ftp.reply_msg" + to: "zeek.ftp.reply.msg" + - from: "zeek.ftp.data_channel.orig_h" + to: "zeek.ftp.data_channel.originating_host" + - from: "zeek.ftp.data_channel.resp_h" + to: "zeek.ftp.data_channel.response_host" + - from: "zeek.ftp.data_channel.resp_p" + to: "zeek.ftp.data_channel.response_port" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.ftp.user", to: "user.name"} + - {from: "zeek.ftp.command", to: "event.action"} + - {from: "zeek.ftp.mime.type", to: "file.mime_type"} + - {from: "zeek.ftp.file.size", to: "file.size"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - info + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: ftp + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/ftp/manifest.yml b/packages/zeek/data_stream/ftp/manifest.yml index dd06d7a58e9..c268d4bf6b9 100644 --- a/packages/zeek/data_stream/ftp/manifest.yml +++ b/packages/zeek/data_stream/ftp/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek ftp.log description: Collect Zeek ftp logs + - input: httpjson + title: Zeek ftp logs via Splunk Enterprise REST API + description: Collect Zeek ftp logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"ftp-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..fde7885c046 --- /dev/null +++ b/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs @@ -0,0 +1,143 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.http + - rename: + fields: + - from: "zeek.http.id.orig_h" + to: "source.address" + - from: "zeek.http.id.orig_p" + to: "source.port" + - from: "zeek.http.id.resp_h" + to: "destination.address" + - from: "zeek.http.id.resp_p" + to: "destination.port" + - from: "zeek.http.uid" + to: "zeek.session_id" + - from: "zeek.http.method" + to: "http.request.method" + - from: "zeek.http.referrer" + to: "http.request.referrer" + - from: "zeek.http.status_code" + to: "http.response.status_code" + - from: "zeek.http.version" + to: "http.version" + - from: "zeek.http.request_body_len" + to: "http.request.body.bytes" + - from: "zeek.http.response_body_len" + to: "http.response.body.bytes" + - from: "zeek.http.uri" + to: "url.original" + - from: "zeek.http.host" + to: "url.domain" + - from: "zeek.http.username" + to: "url.username" + - from: "zeek.http.password" + to: "url.password" + - from: "zeek.http.user_agent" + to: "user_agent.original" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "destination.port", to: "url.port"} + - {from: "http.request.method", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + - web + type: + - connection + - info + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + diff --git a/packages/zeek/data_stream/http/manifest.yml b/packages/zeek/data_stream/http/manifest.yml index a4b9c9fed0a..83f7ec413ec 100644 --- a/packages/zeek/data_stream/http/manifest.yml +++ b/packages/zeek/data_stream/http/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek http.log description: Collect Zeek http logs + - input: httpjson + title: Zeek http logs via Splunk Enterprise REST API + description: Collect Zeek http logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"http-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..d7b9444eb05 --- /dev/null +++ b/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs @@ -0,0 +1,136 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.intel + - convert: + ignore_missing: true + fields: + - {from: zeek.intel.id.orig_h, to: source.address} + - {from: zeek.intel.id.orig_h, to: source.ip, type: ip} + - {from: zeek.intel.id.orig_p, to: source.port, type: long} + - {from: zeek.intel.id.resp_h, to: destination.address} + - {from: zeek.intel.id.resp_h, to: destination.ip, type: ip} + - {from: zeek.intel.id.resp_p, to: destination.port, type: long} + - rename: + ignore_missing: true + fields: + - from: zeek.intel.uid + to: zeek.session_id + # Expand field names containing dots. + - from: zeek.intel.seen.indicator + to: seen.indicator + - from: zeek.intel.seen.indicator_type + to: seen.indicator_type + - from: zeek.intel.seen.host + to: seen.host + - from: zeek.intel.seen.where + to: seen.where + - from: zeek.intel.seen.node + to: seen.node + - from: zeek.intel.seen.conn + to: seen.conn + - from: zeek.intel.seen.uid + to: seen.uid + - from: zeek.intel.seen.f + to: seen.f + - from: zeek.intel.seen.fuid + to: seen.fuid + - from: seen + to: zeek.intel.seen + - drop_fields: + ignore_missing: true + fields: + - zeek.intel.id.orig_h + - zeek.intel.id.orig_p + - zeek.intel.id.resp_h + - zeek.intel.id.resp_p + - add_fields: + target: event + fields: + kind: alert + type: + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/intel/manifest.yml b/packages/zeek/data_stream/intel/manifest.yml index d17790bd300..a84bc9c02cc 100644 --- a/packages/zeek/data_stream/intel/manifest.yml +++ b/packages/zeek/data_stream/intel/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek intel.log description: Collect Zeek intel logs + - input: httpjson + title: Zeek intel logs via Splunk Enterprise REST API + description: Collect Zeek intel logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"intel-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..af7d06f90c4 --- /dev/null +++ b/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs @@ -0,0 +1,132 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.irc + - rename: + fields: + - from: "zeek.irc.id.orig_h" + to: "source.address" + - from: "zeek.irc.id.orig_p" + to: "source.port" + - from: "zeek.irc.id.resp_h" + to: "destination.address" + - from: "zeek.irc.id.resp_p" + to: "destination.port" + - from: "zeek.irc.uid" + to: "zeek.session_id" + - from: "zeek.irc.dcc_file_name" + to: "zeek.irc.dcc.file.name" + - from: "zeek.irc.dcc_file_size" + to: "zeek.irc.dcc.file.size" + - from: "zee.irc.dcc_mime_type" + to: "zeek.irc.dcc.mime_type" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.irc.user", to: "user.name"} + - {from: "zeek.irc.command", to: "event.action"} + - {from: "zeek.irc.dcc.file.name", to: "file.name"} + - {from: "zeek.irc.dcc.file.size", to: "file.size"} + - {from: "zeek.irc.dcc.mime_type", to: "file.mime_type"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: irc + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/irc/manifest.yml b/packages/zeek/data_stream/irc/manifest.yml index ddd2c072b06..c4fd915fd22 100644 --- a/packages/zeek/data_stream/irc/manifest.yml +++ b/packages/zeek/data_stream/irc/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek irc.log description: Collect Zeek irc logs + - input: httpjson + title: Zeek irc logs via Splunk Enterprise REST API + description: Collect Zeek irc logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"irc-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..f8e19e2c2f5 --- /dev/null +++ b/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs @@ -0,0 +1,155 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.kerberos + - rename: + fields: + - from: "zeek.kerberos.id.orig_h" + to: "source.address" + - from: "zeek.kerberos.id.orig_p" + to: "source.port" + - from: "zeek.kerberos.id.resp_h" + to: "destination.address" + - from: "zeek.kerberos.id.resp_p" + to: "destination.port" + - from: "zeek.kerberos.uid" + to: "zeek.session_id" + - from: "zeek.kerberos.till" + to: "zeek.kerberos.valid.until" + - from: "zeek.kerberos.from" + to: "zeek.kerberos.valid.from" + - from: "zeek.kerberos.error_code" + to: "zeek.kerberos.error.code" + - from: "zeek.kerberos.error_msg" + to: "zeek.kerberos.error.msg" + - from: "zeek.kerberos.cert.client" + to: "zeek.kerberos.cert.client.value" + - from: "zeek.kerberos.cert.client_subject" + to: "zeek.kerberos.cert.client.subject" + - from: "zeek.kerberos.cert.client_fuid" + to: "zeek.kerberos.cert.client.fuid" + - from: "zeek.kerberos.cert.server" + to: "zeek.kerberos.cert.server.value" + - from: "zeek.kerberos.cert.server_subject" + to: "zeek.kerberos.cert.server.subject" + - from: "zeek.kerberos.cert.server_fuid" + to: "zeek.kerberos.cert.server.fuid" + - from: "zeek.kerberos.auth_ticket" + to: "zeek.kerberos.ticket.auth" + - from: "zeek.kerberos.new_ticket" + to: "zeek.kerberos.ticket.new" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "source.address", to: "client.address"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "destination.address", to: "server.address"} + - {from: "zeek.kerberos.request_type", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - authentication + - dissect: + when: + contains: + zeek.kerberos.client: "/" + tokenizer: "%{user.name}/%{user.domain}" + field: zeek.kerberos.client + target_prefix: "" + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: kerberos + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/kerberos/manifest.yml b/packages/zeek/data_stream/kerberos/manifest.yml index 683001f6768..8d27a8017a5 100644 --- a/packages/zeek/data_stream/kerberos/manifest.yml +++ b/packages/zeek/data_stream/kerberos/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek kerberos.log description: Collect Zeek kerberos logs + - input: httpjson + title: Zeek kerberos logs via Splunk Enterprise REST API + description: Collect Zeek kerberos logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"kerberos-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..b19225b810b --- /dev/null +++ b/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs @@ -0,0 +1,135 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.modbus + - rename: + fields: + - from: "zeek.modbus.id.orig_h" + to: "source.address" + - from: "zeek.modbus.id.orig_p" + to: "source.port" + - from: "zeek.modbus.id.resp_h" + to: "destination.address" + - from: "zeek.modbus.id.resp_p" + to: "destination.port" + - from: "zeek.modbus.uid" + to: "zeek.session_id" + - from: "zeek.modbus.func" + to: "zeek.modbus.function" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.modbus.function", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - if: + has_fields: ['zeek.modbus.exception'] + then: + - add_fields: + target: event + fields: + outcome: failure + else: + - add_fields: + target: event + fields: + outcome: success + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: modbus + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/modbus/manifest.yml b/packages/zeek/data_stream/modbus/manifest.yml index e67d9e1ae50..138dde5b9c2 100644 --- a/packages/zeek/data_stream/modbus/manifest.yml +++ b/packages/zeek/data_stream/modbus/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek modbus.log description: Collect Zeek modbus logs + - input: httpjson + title: Zeek modbus logs via Splunk Enterprise REST API + description: Collect Zeek modbus logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"modbus-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..ee5d9efc553 --- /dev/null +++ b/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs @@ -0,0 +1,135 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.mysql + - rename: + fields: + - from: "zeek.mysql.id.orig_h" + to: "source.address" + - from: "zeek.mysql.id.orig_p" + to: "source.port" + - from: "zeek.mysql.id.resp_h" + to: "destination.address" + - from: "zeek.mysql.id.resp_p" + to: "destination.port" + - from: "zeek.mysql.uid" + to: "zeek.session_id" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.mysql.cmd", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - database + - network + type: + - connection + - protocol + - if: + equals: + zeek.mysql.success: true + then: + - add_fields: + target: event + fields: + outcome: success + else: + - add_fields: + target: event + fields: + outcome: failure + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: mysql + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/mysql/manifest.yml b/packages/zeek/data_stream/mysql/manifest.yml index 17c091265ef..efd97041c98 100644 --- a/packages/zeek/data_stream/mysql/manifest.yml +++ b/packages/zeek/data_stream/mysql/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek mysql.log description: Collect Zeek mysql logs + - input: httpjson + title: Zeek mysql logs via Splunk Enterprise REST API + description: Collect Zeek mysql logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"mysql-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..06782cc5843 --- /dev/null +++ b/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs @@ -0,0 +1,152 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.notice + - rename: + fields: + - from: "zeek.notice.src" + to: "source.address" + - from: "zeek.notice.dst" + to: "destination.address" + - from: "zeek.notice.uid" + to: "zeek.session_id" + - from: "zeek.notice.p" + to: "destination.port" + - from: "zeek.notice.conn" + to: "zeek.notice.connnection_id" + - from: "zeek.notice.iconn" + to: "zeek.notice.icmp_id" + - from: "zeek.notice.id.orig_h" + to: "source.address" + - from: "zeek.notice.id.orig_p" + to: "source.port" + - from: "zeek.notice.id.resp_h" + to: "destination.address" + - from: "zeek.notice.id.resp_p" + to: "destination.port" + - from: "zeek.notice.proto" + to: "network.transport" + - from: "zeek.notice.id.orig_p" + to: "source.port" + - from: "zeek.notice.f.id" + to: "zeek.notice.file.id" + - from: "zeek.notice.f.parent_id" + to: "zeek.notice.file.parent_id" + - from: "zeek.notice.f.source" + to: "zeek.notice.file.source" + - from: "zeek.notice.f.is_orig" + to: "zeek.notice.file.is_orig" + - from: "zeek.notice.f.seen_bytes" + to: "zeek.notice.file.seen_bytes" + - from: "zeek.notice.f.total_bytes" + to: "zeek.notice.file.total_bytes" + - from: "zeek.notice.file_mime_type" + to: "zeek.notice.file.mime_type" + ignore_missing: true + fail_on_error: false + - drop_fields: + fields: ["zeek.notice.remote_location", "zeek.notice.f"] + ignore_missing: true + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.notice.file.total_bytes", to: "file.size"} + - {from: "zeek.notice.file.mime_type", to: "file.mime_type"} + - {from: "zeek.notice.note", to: "rule.name"} + - {from: "zeek.notice.msg", to: "rule.description"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: alert + category: + - intrusion_detection + type: + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/notice/manifest.yml b/packages/zeek/data_stream/notice/manifest.yml index 1c082686159..d3c72f9181c 100644 --- a/packages/zeek/data_stream/notice/manifest.yml +++ b/packages/zeek/data_stream/notice/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek notice.log description: Collect Zeek notice logs + - input: httpjson + title: Zeek notice logs via Splunk Enterprise REST API + description: Collect Zeek notice logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"notice-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..5a8920648f4 --- /dev/null +++ b/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs @@ -0,0 +1,145 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.ntlm + - rename: + fields: + - from: "zeek.ntlm.id.orig_h" + to: "source.address" + - from: "zeek.ntlm.id.orig_p" + to: "source.port" + - from: "zeek.ntlm.id.resp_h" + to: "destination.address" + - from: "zeek.ntlm.id.resp_p" + to: "destination.port" + - from: "zeek.ntlm.uid" + to: "zeek.session_id" + - from: "zeek.ntlm.domainname" + to: "zeek.ntlm.domain" + - from: "zeek.ntlm.server_dns_computer_name" + to: "zeek.ntlm.server.name.dns" + - from: "zeek.ntlm.server_nb_computer_name" + to: "zeek.ntlm.server.name.netbios" + - from: "zeek.ntlm.server_tree_name" + to: "zeek.ntlm.server.name.tree" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.ntlm.username", to: "user.name"} + - {from: "zeek.ntlm.domain", to: "user.domain"} + - add_fields: + target: event + fields: + kind: event + category: + - authentication + - network + type: + - info + - connection + - if: + equals: + zeek.ntlm.success: true + then: + - add_fields: + target: event + fields: + outcome: success + - if: + equals: + zeek.ntlm.success: false + then: + - add_fields: + target: event + fields: + outcome: failure + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: ntlm + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/ntlm/manifest.yml b/packages/zeek/data_stream/ntlm/manifest.yml index c0c30d8bc7f..5e69cc1494a 100644 --- a/packages/zeek/data_stream/ntlm/manifest.yml +++ b/packages/zeek/data_stream/ntlm/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek ntlm.log description: Collect Zeek ntlm logs + - input: httpjson + title: Zeek ntlm logs via Splunk Enterprise REST API + description: Collect Zeek ntlm logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"ntlm-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..22c009835f9 --- /dev/null +++ b/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs @@ -0,0 +1,120 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.ocsp + - rename: + fields: + - from: "zeek.ocsp.id" + to: "zeek.ocsp.file_id" + - from: "zeek.ocsp.hashAlgorithm" + to: "zeek.ocsp.hash.algorithm" + - from: "zeek.ocsp.issuerNameHash" + to: "zeek.ocsp.hash.issuer.name" + - from: "zeek.ocsp.issuerKeyHash" + to: "zeek.ocsp.hash.issuer.key" + - from: "zeek.ocsp.serialNumber" + to: "zeek.ocsp.serial_number" + - from: "zeek.ocsp.serialNumber" + to: "zeek.ocsp.serial_number" + - from: "zeek.ocsp.certStatus" + to: "zeek.ocsp.status" + - from: "zeek.ocsp.certStatus" + to: "zeek.ocsp.status" + - from: "zeek.ocsp.revoketime" + to: "zeek.ocsp.revoke.date" + - from: "zeek.ocsp.revokereason" + to: "zeek.ocsp.revoke.reason" + - from: "zeek.ocsp.thisUpdate" + to: "zeek.ocsp.update.this" + - from: "zeek.ocsp.nextUpdate" + to: "zeek.ocsp.update.next" + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/ocsp/manifest.yml b/packages/zeek/data_stream/ocsp/manifest.yml index dada6fdc760..65f0be701e9 100644 --- a/packages/zeek/data_stream/ocsp/manifest.yml +++ b/packages/zeek/data_stream/ocsp/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek ocsp.log description: Collect Zeek ocsp logs + - input: httpjson + title: Zeek ocsp logs via Splunk Enterprise REST API + description: Collect Zeek ocsp logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"ocsp-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..31cb372fd40 --- /dev/null +++ b/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs @@ -0,0 +1,101 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.pe + - rename: + fields: + - from: "zeek.pe.compile_ts" + to: "zeek.pe.compile_time" + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - file + type: + - info + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/pe/manifest.yml b/packages/zeek/data_stream/pe/manifest.yml index ff416ced0cd..a905d6db869 100644 --- a/packages/zeek/data_stream/pe/manifest.yml +++ b/packages/zeek/data_stream/pe/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek pe.log description: Collect Zeek pe logs + - input: httpjson + title: Zeek pe logs via Splunk Enterprise REST API + description: Collect Zeek pe logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"pe-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..9ae128edeb9 --- /dev/null +++ b/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs @@ -0,0 +1,121 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.radius + - rename: + fields: + - from: "zeek.radius.id.orig_h" + to: "source.address" + - from: "zeek.radius.id.orig_p" + to: "source.port" + - from: "zeek.radius.id.resp_h" + to: "destination.address" + - from: "zeek.radius.id.resp_p" + to: "destination.port" + - from: "zeek.radius.uid" + to: "zeek.session_id" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.radius.username", to: "user.name"} + - {from: "zeek.radius.result", to: "event.outcome"} + - add_fields: + target: event + fields: + kind: event + category: + - authentication + - network + type: + - info + - connection + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: udp + netwokr.protocol: radius + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/radius/manifest.yml b/packages/zeek/data_stream/radius/manifest.yml index 800410ddc9a..05e9be37c72 100644 --- a/packages/zeek/data_stream/radius/manifest.yml +++ b/packages/zeek/data_stream/radius/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek radius.log description: Collect Zeek radius logs + - input: httpjson + title: Zeek radius logs via Splunk Enterprise REST API + description: Collect Zeek radius logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"radius-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..640167940a9 --- /dev/null +++ b/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs @@ -0,0 +1,140 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.rdp + - rename: + fields: + - from: "zeek.rdp.id.orig_h" + to: "source.address" + - from: "zeek.rdp.id.orig_p" + to: "source.port" + - from: "zeek.rdp.id.resp_h" + to: "destination.address" + - from: "zeek.rdp.id.resp_p" + to: "destination.port" + - from: "zeek.rdp.uid" + to: "zeek.session_id" + - from: "zeek.rdp.client_build" + to: "zeek.rdp.client.build" + - from: "zeek.rdp.client_name" + to: "zeek.rdp.client.name" + - from: "zeek.rdp.client_dig_product_id" + to: "zeek.rdp.client.product_id" + - from: "zeek.rdp.desktop_width" + to: "zeek.rdp.desktop.width" + - from: "zeek.rdp.desktop_height" + to: "zeek.rdp.desktop.height" + - from: "zeek.rdp.requested_color_depth" + to: "zeek.rdp.desktop.color_depth" + - from: "zeek.rdp.cert_type" + to: "zeek.rdp.cert.type" + - from: "zeek.rdp.cert_count" + to: "zeek.rdp.cert.count" + - from: "zeek.rdp.cert_permanent" + to: "zeek.rdp.cert.permanent" + - from: "zeek.rdp.encryption_level" + to: "zeek.rdp.encryption.level" + - from: "zeek.rdp.encryption_method" + to: "zeek.rdp.encryption.method" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - protocol + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: rdp + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/rdp/manifest.yml b/packages/zeek/data_stream/rdp/manifest.yml index 2324e71fbb0..5af506eeaf8 100644 --- a/packages/zeek/data_stream/rdp/manifest.yml +++ b/packages/zeek/data_stream/rdp/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek rdp.log description: Collect Zeek rdp logs + - input: httpjson + title: Zeek rdp logs via Splunk Enterprise REST API + description: Collect Zeek rdp logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"rdp-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..4873f73e76e --- /dev/null +++ b/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs @@ -0,0 +1,130 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.rfb + - rename: + fields: + - from: "zeek.rfb.id.orig_h" + to: "source.address" + - from: "zeek.rfb.id.orig_p" + to: "source.port" + - from: "zeek.rfb.id.resp_h" + to: "destination.address" + - from: "zeek.rfb.id.resp_p" + to: "destination.port" + - from: "zeek.rfb.uid" + to: "zeek.session_id" + - from: "zeek.rfb.client_major_version" + to: "zeek.rfb.version.client.major" + - from: "zeek.rfb.client_minor_version" + to: "zeek.rfb.version.client.minor" + - from: "zeek.rfb.server_major_version" + to: "zeek.rfb.version.server.major" + - from: "zeek.rfb.server_minor_version" + to: "zeek.rfb.version.server.minor" + - from: "zeek.rfb.auth" + to: "zeek.rfb.auth.success" + - from: "zeek.rfb.authentication_method" + to: "zeek.rfb.auth.method" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - info + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: rfb + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/rfb/manifest.yml b/packages/zeek/data_stream/rfb/manifest.yml index e71d92bd01e..b7a1a13f715 100644 --- a/packages/zeek/data_stream/rfb/manifest.yml +++ b/packages/zeek/data_stream/rfb/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek rfb.log description: Collect Zeek rfb logs + - input: httpjson + title: Zeek rfb logs via Splunk Enterprise REST API + description: Collect Zeek rfb logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"rfb-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..2b5f1ce8806 --- /dev/null +++ b/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs @@ -0,0 +1,146 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.sip + - rename: + fields: + - from: "zeek.sip.id.orig_h" + to: "source.address" + - from: "zeek.sip.id.orig_p" + to: "source.port" + - from: "zeek.sip.id.resp_h" + to: "destination.address" + - from: "zeek.sip.id.resp_p" + to: "destination.port" + - from: "zeek.sip.uid" + to: "zeek.session_id" + - from: "zeek.sip.trans_depth" + to: "zeek.sip.transaction_depth" + - from: "zeek.sip.method" + to: "zeek.sip.sequence.method" + - from: "zeek.sip.request_from" + to: "zeek.sip.request.from" + - from: "zeek.sip.request_to" + to: "zeek.sip.request.to" + - from: "zeek.sip.request_path" + to: "zeek.sip.request.path" + - from: "zeek.sip.request_body_len" + to: "zeek.sip.request.body_length" + - from: "zeek.sip.response_from" + to: "zeek.sip.response.from" + - from: "zeek.sip.response_to" + to: "zeek.sip.response.to" + - from: "zeek.sip.response_path" + to: "zeek.sip.response.path" + - from: "zeek.sip.response_body_len" + to: "zeek.sip.response.body_length" + - from: "zeek.sip.status_code" + to: "zeek.sip.status.code" + - from: "zeek.sip.status_msg" + to: "zeek.sip.status.msg" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.sip.sequence.method", to: "event.action"} + - {from: "zeek.sip.uri", to: "url.full"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: udp + network.protocol: sip + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/sip/manifest.yml b/packages/zeek/data_stream/sip/manifest.yml index d35bb8951dc..ac179588f38 100644 --- a/packages/zeek/data_stream/sip/manifest.yml +++ b/packages/zeek/data_stream/sip/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek sip.log description: Collect Zeek sip logs + - input: httpjson + title: Zeek sip logs via Splunk Enterprise REST API + description: Collect Zeek sip logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"sip-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..dc1c6886775 --- /dev/null +++ b/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs @@ -0,0 +1,154 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.smb_cmd + - drop_fields: + fields: + - "zeek.smb_cmd.referenced_file.ts" + - "zeek.smb_cmd.referenced_file.id.orig_p" + - "zeek.smb_cmd.referenced_file.id.resp_p" + - "zeek.smb_cmd.referenced_file.size" + - "zeek.smb_cmd.referenced_file.times.modified" + - "zeek.smb_cmd.referenced_file.times.accessed" + - "zeek.smb_cmd.referenced_file.times.created" + - "zeek.smb_cmd.referenced_file.times.changed" + ignore_missing: true + - drop_fields: + when: + not: + has_fields: ["zeek.smb_cmd.referenced_file.action"] + fields: + - "zeek.smb_cmd.referenced_file.uid" + - "zeek.smb_cmd.referenced_file.id.orig_h" + - "zeek.smb_cmd.referenced_file.id.resp_h" + ignore_missing: true + - rename: + fields: + - from: "zeek.smb_cmd.id.orig_h" + to: "source.address" + - from: "zeek.smb_cmd.id.orig_p" + to: "source.port" + - from: "zeek.smb_cmd.id.resp_h" + to: "destination.address" + - from: "zeek.smb_cmd.id.resp_p" + to: "destination.port" + - from: "zeek.smb_cmd.uid" + to: "zeek.session_id" + - from: "zeek.smb_cmd.referenced_file.uid" + to: "zeek.smb_cmd.file.uid" + - from: "zeek.smb_cmd.referenced_file.id.orig_h" + to: "zeek.smb_cmd.file.host.tx" + - from: "zeek.smb_cmd.referenced_file.id.resp_h" + to: "zeek.smb_cmd.file.host.rx" + - from: "zeek.smb_cmd.referenced_file.name" + to: "zeek.smb_cmd.file.name" + - from: "zeek.smb_cmd.referenced_file.path" + to: "zeek.smb_cmd.file.path" + - from: "zeek.smb_cmd.referenced_file.action" + to: "zeek.smb_cmd.file.action" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.smb_cmd.command", to: "event.action"} + - {from: "zeek.smb_cmd.username", to: "user.name"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: smb + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/smb_cmd/manifest.yml b/packages/zeek/data_stream/smb_cmd/manifest.yml index 8905a97bc87..566d21a615c 100644 --- a/packages/zeek/data_stream/smb_cmd/manifest.yml +++ b/packages/zeek/data_stream/smb_cmd/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek smb_cmd.log description: Collect Zeek smb_cmd logs + - input: httpjson + title: Zeek smb_cmd logs via Splunk Enterprise REST API + description: Collect Zeek smb_cmd logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"smb_cmd-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..17a9d8db944 --- /dev/null +++ b/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs @@ -0,0 +1,124 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.smb_files + - rename: + fields: + - from: "zeek.smb_files.id.orig_h" + to: "source.address" + - from: "zeek.smb_files.id.orig_p" + to: "source.port" + - from: "zeek.smb_files.id.resp_h" + to: "destination.address" + - from: "zeek.smb_files.id.resp_p" + to: "destination.port" + - from: "zeek.smb_files.uid" + to: "zeek.session_id" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.smb_files.action", to: "event.action"} + - {from: "zeek.smb_files.name", to: "file.name"} + - {from: "zeek.smb_files.size", to: "file.size"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + - file + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: smb + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/smb_files/manifest.yml b/packages/zeek/data_stream/smb_files/manifest.yml index 34e8213cc8b..5498d3e0904 100644 --- a/packages/zeek/data_stream/smb_files/manifest.yml +++ b/packages/zeek/data_stream/smb_files/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek smb_files.log description: Collect Zeek smb_files logs + - input: httpjson + title: Zeek smb_files logs via Splunk Enterprise REST API + description: Collect Zeek smb_files logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"smb_files-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..0a7b1cf61b2 --- /dev/null +++ b/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs @@ -0,0 +1,120 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.smb_mapping + - rename: + fields: + - from: "zeek.smb_mapping.id.orig_h" + to: "source.address" + - from: "zeek.smb_mapping.id.orig_p" + to: "source.port" + - from: "zeek.smb_mapping.id.resp_h" + to: "destination.address" + - from: "zeek.smb_mapping.id.resp_p" + to: "destination.port" + - from: "zeek.smb_mapping.uid" + to: "zeek.session_id" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: smb + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/smb_mapping/manifest.yml b/packages/zeek/data_stream/smb_mapping/manifest.yml index e92ebc04435..db619894725 100644 --- a/packages/zeek/data_stream/smb_mapping/manifest.yml +++ b/packages/zeek/data_stream/smb_mapping/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek smb_mapping.log description: Collect Zeek smb_mapping logs + - input: httpjson + title: Zeek smb_mapping logs via Splunk Enterprise REST API + description: Collect Zeek smb_mapping logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"smb_mapping-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..b2b9173ba31 --- /dev/null +++ b/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs @@ -0,0 +1,127 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.smtp + - rename: + fields: + - from: "zeek.smtp.id.orig_h" + to: "source.address" + - from: "zeek.smtp.id.orig_p" + to: "source.port" + - from: "zeek.smtp.id.resp_h" + to: "destination.address" + - from: "zeek.smtp.id.resp_p" + to: "destination.port" + - from: "zeek.smtp.uid" + to: "zeek.session_id" + - from: "zeek.smtp.trans_depth" + to: "zeek.smtp.transaction_depth" + - from: "zeek.smtp.mailfrom" + to: "zeek.smtp.mail_from" + - from: "zeek.smtp.rcptto" + to: "zeek.smtp.rcpt_to" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.smtp.tls", to: "tls.established", type: boolean} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: smtp + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/smtp/manifest.yml b/packages/zeek/data_stream/smtp/manifest.yml index 26c04cca4e1..c439660f65b 100644 --- a/packages/zeek/data_stream/smtp/manifest.yml +++ b/packages/zeek/data_stream/smtp/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek smtp.log description: Collect Zeek smtp logs + - input: httpjson + title: Zeek smtp logs via Splunk Enterprise REST API + description: Collect Zeek smtp logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"smtp-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..e6465dcc0df --- /dev/null +++ b/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs @@ -0,0 +1,128 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.snmp + - rename: + fields: + - from: "zeek.snmp.id.orig_h" + to: "source.address" + - from: "zeek.snmp.id.orig_p" + to: "source.port" + - from: "zeek.snmp.id.resp_h" + to: "destination.address" + - from: "zeek.snmp.id.resp_p" + to: "destination.port" + - from: "zeek.snmp.uid" + to: "event.id" + - from: "zeek.snmp.get_requests" + to: "zeek.snmp.get.requests" + - from: "zeek.snmp.get_bulk_requests" + to: "zeek.snmp.get.bulk_requests" + - from: "zeek.snmp.get_responses" + to: "zeek.snmp.get.responses" + - from: "zeek.snmp.set_requests" + to: "zeek.snmp.set.requests" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: udp + network.protocol: snmp + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/snmp/manifest.yml b/packages/zeek/data_stream/snmp/manifest.yml index e6ca5f511f9..84577df53fc 100644 --- a/packages/zeek/data_stream/snmp/manifest.yml +++ b/packages/zeek/data_stream/snmp/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek snmp.log description: Collect Zeek snmp logs + - input: httpjson + title: Zeek snmp logs via Splunk Enterprise REST API + description: Collect Zeek snmp logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"snmp-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..9be356a6775 --- /dev/null +++ b/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs @@ -0,0 +1,127 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.socks + - rename: + fields: + - from: "zeek.socks.id.orig_h" + to: "source.address" + - from: "zeek.socks.id.orig_p" + to: "source.port" + - from: "zeek.socks.id.resp_h" + to: "destination.address" + - from: "zeek.socks.id.resp_p" + to: "destination.port" + - from: "zeek.socks.uid" + to: "zeek.session_id" + - from: "zeek.socks.request.name" + to: "zeek.socks.request.host" + - from: "zeek.socks.request_p" + to: "zeek.socks.request.port" + - from: "zeek.socks.bound_p" + to: "zeek.socks.bound.port" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.socks.user", to: "user.name"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: socks + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/socks/manifest.yml b/packages/zeek/data_stream/socks/manifest.yml index e1bad6d1a20..78a6875a912 100644 --- a/packages/zeek/data_stream/socks/manifest.yml +++ b/packages/zeek/data_stream/socks/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek socks.log description: Collect Zeek socks logs + - input: httpjson + title: Zeek socks logs via Splunk Enterprise REST API + description: Collect Zeek socks logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"socks-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..f9c7f36e4a2 --- /dev/null +++ b/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs @@ -0,0 +1,132 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.ssh + - rename: + fields: + - from: "zeek.ssh.id.orig_h" + to: "source.address" + - from: "zeek.ssh.id.orig_p" + to: "source.port" + - from: "zeek.ssh.id.resp_h" + to: "destination.address" + - from: "zeek.ssh.id.resp_p" + to: "destination.port" + - from: "zeek.ssh.uid" + to: "zeek.session_id" + - from: "zeek.ssh.auth_attempts" + to: "zeek.ssh.auth.attempts" + - from: "zeek.ssh.auth_success" + to: "zeek.ssh.auth.success" + - from: "zeek.ssh.cipher_alg" + to: "zeek.ssh.algorithm.cipher" + - from: "zeek.ssh.mac_alg" + to: "zeek.ssh.algorithm.mac" + - from: "zeek.ssh.compression_alg" + to: "zeek.ssh.algorithm.compression" + - from: "zeek.ssh.kex_alg" + to: "zeek.ssh.algorithm.key_exchange" + - from: "zeek.ssh.host_key_alg" + to: "zeek.ssh.algorithm.host_key" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + network.protocol: ssh + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/ssh/manifest.yml b/packages/zeek/data_stream/ssh/manifest.yml index 777ddb20a19..10f7f924706 100644 --- a/packages/zeek/data_stream/ssh/manifest.yml +++ b/packages/zeek/data_stream/ssh/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek ssh.log description: Collect Zeek ssh logs + - input: httpjson + title: Zeek ssh logs via Splunk Enterprise REST API + description: Collect Zeek ssh logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"ssh-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..45b2f0a9941 --- /dev/null +++ b/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs @@ -0,0 +1,135 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.ssl + - rename: + fields: + - from: "zeek.ssl.id.orig_h" + to: "source.address" + - from: "zeek.ssl.id.orig_p" + to: "source.port" + - from: "zeek.ssl.id.resp_h" + to: "destination.address" + - from: "zeek.ssl.id.resp_p" + to: "destination.port" + - from: "zeek.ssl.uid" + to: "zeek.session_id" + - from: "zeek.ssl.server_name" + to: "zeek.ssl.server.name" + - from: "zeek.ssl.cert_chain" + to: "zeek.ssl.server.cert_chain" + - from: "zeek.ssl.cert_chain_fuids" + to: "zeek.ssl.server.cert_chain_fuids" + - from: "zeek.ssl.client_cert_chain" + to: "zeek.ssl.client.cert_chain" + - from: "zeek.ssl.client_cert_chain_fuids" + to: "zeek.ssl.client.cert_chain_fuids" + - from: "zeek.ssl.validation_status" + to: "zeek.ssl.validation.status" + - from: "zeek.ssl.validation_code" + to: "zeek.ssl.validation.code" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "source.address", to: "client.address"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "destination.address", to: "server.address"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.transport: tcp + - drop_fields: + fields: ["json","message"] + ignore_missing: true \ No newline at end of file diff --git a/packages/zeek/data_stream/ssl/manifest.yml b/packages/zeek/data_stream/ssl/manifest.yml index cb28b2168e5..385b78b4aab 100644 --- a/packages/zeek/data_stream/ssl/manifest.yml +++ b/packages/zeek/data_stream/ssl/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek ssl.log description: Collect Zeek ssl logs + - input: httpjson + title: Zeek ssl logs via Splunk Enterprise REST API + description: Collect Zeek ssl logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"ssl-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..ad308f0fccd --- /dev/null +++ b/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs @@ -0,0 +1,141 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.stats + - rename: + fields: + - from: "zeek.stats.mem" + to: "zeek.stats.memory" + - from: "zeek.stats.pkts_proc" + to: "zeek.stats.packets.processed" + - from: "zeek.stats.pkts_dropped" + to: "zeek.stats.packets.dropped" + - from: "zeek.stats.pkts_link" + to: "zeek.stats.packets.received" + - from: "zeek.stats.pkts_link" + to: "zeek.stats.packets.received" + - from: "zeek.stats.bytes_recv" + to: "zeek.stats.bytes.received" + - from: "zeek.stats.tcp_conns" + to: "zeek.stats.connections.tcp.count" + - from: "zeek.stats.active_tcp_conns" + to: "zeek.stats.connections.tcp.active" + - from: "zeek.stats.udp_conns" + to: "zeek.stats.connections.udp.count" + - from: "zeek.stats.active_udp_conns" + to: "zeek.stats.connections.udp.active" + - from: "zeek.stats.icmp_conns" + to: "zeek.stats.connections.icmp.count" + - from: "zeek.stats.active_icmp_conns" + to: "zeek.stats.connections.icmp.active" + - from: "zeek.stats.events_proc" + to: "zeek.stats.events.processed" + - from: "zeek.stats.events_queued" + to: "zeek.stats.events.queued" + - from: "zeek.stats.timers" + to: "zeek.stats.timers.count" + - from: "zeek.stats.active_timers" + to: "zeek.stats.timers.active" + - from: "zeek.stats.files" + to: "zeek.stats.files.count" + - from: "zeek.stats.active_files" + to: "zeek.stats.files.active" + - from: "zeek.stats.dns_requests" + to: "zeek.stats.dns_requests.count" + - from: "zeek.stats.active_dns_requests" + to: "zeek.stats.dns_requests.active" + - from: "zeek.stats.reassem_tcp_size" + to: "zeek.stats.reassembly_size.tcp" + - from: "zeek.stats.reassem_file_size" + to: "zeek.stats.reassembly_size.file" + - from: "zeek.stats.reassem_frag_size" + to: "zeek.stats.reassembly_size.frag" + - from: "zeek.stats.reassem_unknown_size" + to: "zeek.stats.reassembly_size.unknown" + - from: "zeek.stats.pkt_lag" + to: "zeek.stats.timestamp_lag" + ignore_missing: true + fail_on_error: false + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/stats/manifest.yml b/packages/zeek/data_stream/stats/manifest.yml index 0dcaf696541..9fd576531af 100644 --- a/packages/zeek/data_stream/stats/manifest.yml +++ b/packages/zeek/data_stream/stats/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek stats.log description: Collect Zeek stats logs + - input: httpjson + title: Zeek stats logs via Splunk Enterprise REST API + description: Collect Zeek stats logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"stats-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..de60140037b --- /dev/null +++ b/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs @@ -0,0 +1,118 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.syslog + - rename: + fields: + - from: "zeek.syslog.id.orig_h" + to: "source.address" + - from: "zeek.syslog.id.orig_p" + to: "source.port" + - from: "zeek.syslog.id.resp_h" + to: "destination.address" + - from: "zeek.syslog.id.resp_p" + to: "destination.port" + - from: "zeek.syslog.uid" + to: "zeek.session_id" + - from: "zeek.syslog.proto" + to: "network.transport" + - from: "zeek.syslog.message" + to: "zeek.syslog.msg" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.syslog.facility", to: "log.syslog.facility.name"} + - {from: "zeek.syslog.severity", to: "log.syslog.severity.name"} + - add_fields: + target: event + fields: + kind: event + - community_id: + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + network.protocol: syslog + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/syslog/manifest.yml b/packages/zeek/data_stream/syslog/manifest.yml index 295fdcb252c..d7ca1a4f0a8 100644 --- a/packages/zeek/data_stream/syslog/manifest.yml +++ b/packages/zeek/data_stream/syslog/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek syslog.log description: Collect Zeek syslog logs + - input: httpjson + title: Zeek syslog logs via Splunk Enterprise REST API + description: Collect Zeek syslog logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"syslog-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..b95ee68a624 --- /dev/null +++ b/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs @@ -0,0 +1,111 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.traceroute + - rename: + fields: + - from: "zeek.traceroute.src" + to: "source.address" + - from: "zeek.traceroute.dst" + to: "destination.address" + - from: "zeek.traceroute.proto" + to: "network.transport" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - info + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/traceroute/manifest.yml b/packages/zeek/data_stream/traceroute/manifest.yml index 97eec185210..d4eccbd8e47 100644 --- a/packages/zeek/data_stream/traceroute/manifest.yml +++ b/packages/zeek/data_stream/traceroute/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek traceroute.log description: Collect Zeek traceroute logs + - input: httpjson + title: Zeek traceroute logs via Splunk Enterprise REST API + description: Collect Zeek traceroute logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"traceroute-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..0428351791f --- /dev/null +++ b/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs @@ -0,0 +1,119 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.tunnel + - rename: + fields: + - from: "zeek.tunnel.id.orig_h" + to: "source.address" + - from: "zeek.tunnel.id.orig_p" + to: "source.port" + - from: "zeek.tunnel.id.resp_h" + to: "destination.address" + - from: "zeek.tunnel.id.resp_p" + to: "destination.port" + - from: "zeek.tunnel.uid" + to: "zeek.session_id" + - from: "zeek.tunnel.tunnel_type" + to: "zeek.tunnel.type" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.tunnel.action", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/tunnel/manifest.yml b/packages/zeek/data_stream/tunnel/manifest.yml index c6c97e4fe88..262e74ee473 100644 --- a/packages/zeek/data_stream/tunnel/manifest.yml +++ b/packages/zeek/data_stream/tunnel/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek tunnel.log description: Collect Zeek tunnel logs + - input: httpjson + title: Zeek tunnel logs via Splunk Enterprise REST API + description: Collect Zeek tunnel logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"tunnel-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..ce1c305a4f0 --- /dev/null +++ b/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs @@ -0,0 +1,119 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.weird + - rename: + fields: + - from: "zeek.weird.id.orig_h" + to: "source.address" + - from: "zeek.weird.id.orig_p" + to: "source.port" + - from: "zeek.weird.id.resp_h" + to: "destination.address" + - from: "zeek.weird.id.resp_p" + to: "destination.port" + - from: "zeek.weird.uid" + to: "zeek.session_id" + - from: "zeek.weird.addl" + to: "zeek.weird.additional_info" + ignore_missing: true + fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.weird.name", to: "rule.name"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: alert + category: + - network + type: + - info + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/weird/manifest.yml b/packages/zeek/data_stream/weird/manifest.yml index be4609cade3..008d36e800b 100644 --- a/packages/zeek/data_stream/weird/manifest.yml +++ b/packages/zeek/data_stream/weird/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek weird.log description: Collect Zeek weird logs + - input: httpjson + title: Zeek weird logs via Splunk Enterprise REST API + description: Collect Zeek weird logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"weird-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..147c052fe04 --- /dev/null +++ b/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs @@ -0,0 +1,123 @@ +config_version: 2 +interval: {{interval}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +{{#if ssl.enabled}} +request.url: https://{{server}}:{{port}}/services/search/jobs/export +{{#if ssl.verification_mode}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorites: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +{{else}} +request.url: http://{{server}}:{{port}}/services/search/jobs/export +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +processors: + - decode_json_fields: + fields: message + target: json + add_error_key: true + - drop_event: + when: + not: + has_fields: ['json.result'] + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: "@metadata._id" + - rename: + fields: + - from: json.result._raw + to: event.original + - from: json.result.host + to: host.name + - from: json.result.source + to: file.path + - decode_json_fields: + fields: event.original + target: zeek.x509 + - rename: + fields: + - from: "zeek.x509.id" + to: "zeek.session_id" + - from: "zeek.x509.certificate.not_valid_before" + to: "zeek.x509.certificate.valid.from" + - from: "zeek.x509.certificate.not_valid_after" + to: "zeek.x509.certificate.valid.until" + - from: "zeek.x509.basic_constraints.ca" + to: "zeek.x509.basic_constraints.certificate_authority" + - from: "zeek.x509.basic_constraints.path_len" + to: "zeek.x509.basic_constraints.path_length" + - from: "zeek.x509.certificate.cn" + to: "zeek.x509.certificate.common_name" + - from: "zeek.x509.certificate.issuer" + to: "zeek.x509.certificate.iss" + - from: "zeek.x509.certificate.subject" + to: "zeek.x509.certificate.sub" + - from: "zeek.x509.certificate.key_alg" + to: "zeek.x509.certificate.key.algorithm" + - from: "zeek.x509.certificate.key_length" + to: "zeek.x509.certificate.key.length" + - from: "zeek.x509.certificate.key_type" + to: "zeek.x509.certificate.key.type" + - from: "zeek.x509.certificate.sig_alg" + to: "zeek.x509.certificate.signature_algorithm" + - from: "zeek.x509.logcert" + to: "zeek.x509.log_cert" + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + type: + - info + - add_fields: + target: '' + fields: + ecs.version: 1.7.0 + - drop_fields: + fields: ["json","message"] + ignore_missing: true diff --git a/packages/zeek/data_stream/x509/manifest.yml b/packages/zeek/data_stream/x509/manifest.yml index fa3f70da5f7..6324dadde7e 100644 --- a/packages/zeek/data_stream/x509/manifest.yml +++ b/packages/zeek/data_stream/x509/manifest.yml @@ -23,3 +23,29 @@ streams: template_path: log.yml.hbs title: Zeek x509.log description: Collect Zeek x509 logs + - input: httpjson + title: Zeek x509 logs via Splunk Enterprise REST API + description: Collect Zeek x509 logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"x509-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index 6991da031da..c7e10151c0e 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek -version: 0.5.1 +version: 0.6.0 release: beta description: Zeek Integration type: integration @@ -13,7 +13,7 @@ format_version: 1.0.0 license: basic categories: [network, monitoring, security] conditions: - kibana.version: ^7.11.0 + kibana.version: '^7.12.0' screenshots: - src: /img/kibana-zeek.png title: kibana zeek @@ -39,5 +39,54 @@ policy_templates: - /var/log/bro/current - /opt/zeek/logs/current - /usr/local/var/spool/zeek + - type: httpjson + title: Collect Zeek logs from third-party REST API + description: "Collects Zeek logs from third-party REST API. Supported logs include: connection" + vars: + - name: server + type: text + title: Address of Splunk Enterprise Server + description: hostname or IP + show_user: true + required: true + default: server.example.com + - name: port + type: text + title: Port number of Splunk Enterprise REST API + show_user: true + required: true + default: 8089 + - name: username + type: text + title: Splunk REST API Username + show_user: true + required: true + - name: password + type: password + title: Splunk REST API Password + required: true + show_user: true + - name: ssl.enabled + type: bool + title: SSL enabled + multi: false + required: false + show_user: false + default: true + - name: ssl.verification_mode + type: text + title: Mode of server verification + description: "valid values: none, strict, certificate or full" + multi: false + required: false + show_user: false + default: full + - name: ssl.certificate_authorities + type: text + title: List of root certificates for TLS server verification + description: PEM encoded + multi: true + required: false + show_user: false owner: github: elastic/security-external-integrations From ce669958a87eaa1eb7497fa65d2d01a86bccbbfd Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Tue, 9 Mar 2021 10:09:29 -0600 Subject: [PATCH 2/2] Incorporate feedback --- .../access/agent/stream/httpjson.yml.hbs | 19 +++------ .../error/agent/stream/httpjson.yml.hbs | 19 +++------ packages/apache/manifest.yml | 41 +++++-------------- .../cloudtrail/agent/stream/httpjson.yml.hbs | 16 ++------ packages/aws/manifest.yml | 41 +++++-------------- .../access/agent/stream/httpjson.yml.hbs | 16 ++------ .../error/agent/stream/httpjson.yml.hbs | 16 ++------ packages/nginx/manifest.yml | 41 +++++-------------- .../agent/stream/httpjson.yml.hbs | 16 ++------ .../connection/agent/stream/httpjson.yml.hbs | 16 ++------ .../dce_rpc/agent/stream/httpjson.yml.hbs | 16 ++------ .../dhcp/agent/stream/httpjson.yml.hbs | 18 ++------ .../dnp3/agent/stream/httpjson.yml.hbs | 16 ++------ .../dns/agent/stream/httpjson.yml.hbs | 16 ++------ .../dpd/agent/stream/httpjson.yml.hbs | 16 ++------ .../files/agent/stream/httpjson.yml.hbs | 16 ++------ .../ftp/agent/stream/httpjson.yml.hbs | 18 ++------ .../http/agent/stream/httpjson.yml.hbs | 16 ++------ .../intel/agent/stream/httpjson.yml.hbs | 16 ++------ .../irc/agent/stream/httpjson.yml.hbs | 18 ++------ .../kerberos/agent/stream/httpjson.yml.hbs | 18 ++------ .../modbus/agent/stream/httpjson.yml.hbs | 18 ++------ .../mysql/agent/stream/httpjson.yml.hbs | 18 ++------ .../notice/agent/stream/httpjson.yml.hbs | 18 ++------ .../ntlm/agent/stream/httpjson.yml.hbs | 18 ++------ .../ocsp/agent/stream/httpjson.yml.hbs | 18 ++------ .../pe/agent/stream/httpjson.yml.hbs | 18 ++------ .../radius/agent/stream/httpjson.yml.hbs | 18 ++------ .../rdp/agent/stream/httpjson.yml.hbs | 18 ++------ .../rfb/agent/stream/httpjson.yml.hbs | 18 ++------ .../sip/agent/stream/httpjson.yml.hbs | 16 ++------ .../smb_cmd/agent/stream/httpjson.yml.hbs | 18 ++------ .../smb_files/agent/stream/httpjson.yml.hbs | 18 ++------ .../smb_mapping/agent/stream/httpjson.yml.hbs | 18 ++------ .../smtp/agent/stream/httpjson.yml.hbs | 16 ++------ .../snmp/agent/stream/httpjson.yml.hbs | 18 ++------ .../socks/agent/stream/httpjson.yml.hbs | 16 ++------ .../ssh/agent/stream/httpjson.yml.hbs | 16 ++------ .../ssl/agent/stream/httpjson.yml.hbs | 18 ++------ .../stats/agent/stream/httpjson.yml.hbs | 16 ++------ .../syslog/agent/stream/httpjson.yml.hbs | 16 ++------ .../traceroute/agent/stream/httpjson.yml.hbs | 16 ++------ .../tunnel/agent/stream/httpjson.yml.hbs | 16 ++------ .../weird/agent/stream/httpjson.yml.hbs | 16 ++------ .../x509/agent/stream/httpjson.yml.hbs | 16 ++------ packages/zeek/manifest.yml | 41 +++++-------------- 46 files changed, 188 insertions(+), 690 deletions(-) diff --git a/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs b/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs index c0c14d398d7..47023a8425a 100644 --- a/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs +++ b/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs @@ -5,25 +5,16 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: - set: target: url.params.search - value: {{search}} | streamstats max(_indextime) AS max_indextime + value: |- + {{search}} | streamstats max(_indextime) AS max_indextime - set: target: url.params.output_mode value: "json" diff --git a/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs b/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs index 021204e69b1..527f069bfcc 100644 --- a/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs +++ b/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs @@ -5,25 +5,16 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: - set: target: url.params.search - value: {{search}} | streamstats max(_indextime) AS max_indextime + value: |- + {{search}} | streamstats max(_indextime) AS max_indextime - set: target: url.params.output_mode value: "json" diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml index 20a324fa20b..f5c9c1b61ae 100644 --- a/packages/apache/manifest.yml +++ b/packages/apache/manifest.yml @@ -33,22 +33,16 @@ policy_templates: title: Collect logs from Apache instances description: Collecting Apache access and error logs - type: httpjson - title: Collect Apache logs from third-party REST API - description: Collecting Apache logs via third-party REST API + title: Collect logs from third-party REST API (experimental) + description: Collect logs from third-party REST API (experimental) vars: - - name: server + - name: url type: text - title: Address of Splunk Enterprise Server - description: hostname or IP + title: URL of Splunk Enterprise Server + description: i.e. scheme://host:port, path is automatic show_user: true required: true - default: server.example.com - - name: port - type: text - title: Port number of Splunk Enterprise REST API - show_user: true - required: true - default: 8089 + default: https://server.example.com:8089 - name: username type: text title: Splunk REST API Username @@ -59,28 +53,13 @@ policy_templates: title: Splunk REST API Password required: true show_user: true - - name: ssl.enabled - type: bool - title: SSL enabled + - name: ssl + type: yaml + title: SSL Configuration + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. multi: false required: false show_user: false - default: true - - name: ssl.verification_mode - type: text - title: Mode of server verification - description: "valid values: none, strict, certificate or full" - multi: false - required: false - show_user: false - default: full - - name: ssl.certificate_authorities - type: text - title: List of root certificates for TLS server verification - description: PEM encoded - multi: true - required: false - show_user: false - type: apache/metrics title: Collect metrics from Apache instances description: Collecting Apache status metrics diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs index 9de7d5a067e..102625299b4 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 1f10d7ec5f7..e2aef79c92e 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -240,22 +240,16 @@ policy_templates: default: "amazonaws.com" description: URL of the entry point for an AWS web service. - type: httpjson - title: Collect AWS logs from third-party REST API - description: "Collects AWS logs from third-party REST API" + title: Collect logs from third-party REST API (experimental) + description: Collect logs from third-party REST API (experimental) vars: - - name: server + - name: url type: text - title: Address of Splunk Enterprise Server - description: hostname or IP + title: URL of Splunk Enterprise Server + description: i.e. scheme://host:port, path is automatic show_user: true required: true - default: server.example.com - - name: port - type: text - title: Port number of Splunk Enterprise REST API - show_user: true - required: true - default: 8089 + default: https://server.example.com:8089 - name: username type: text title: Splunk REST API Username @@ -266,27 +260,12 @@ policy_templates: title: Splunk REST API Password required: true show_user: true - - name: ssl.enabled - type: bool - title: SSL enabled - multi: false - required: false - show_user: false - default: true - - name: ssl.verification_mode - type: text - title: Mode of server verification - description: "valid values: none, strict, certificate or full" + - name: ssl + type: yaml + title: SSL Configuration multi: false required: false show_user: false - default: full - - name: ssl.certificate_authorities - type: text - title: List of root certificates for TLS server verification - description: PEM encoded - multi: true - required: false - show_user: false + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. owner: github: elastic/integrations diff --git a/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs b/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs index 7bcbe024e21..588ed579348 100644 --- a/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs +++ b/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs b/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs index 7bcbe024e21..588ed579348 100644 --- a/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs +++ b/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/nginx/manifest.yml b/packages/nginx/manifest.yml index 4a6a279a572..8bade9bd91b 100644 --- a/packages/nginx/manifest.yml +++ b/packages/nginx/manifest.yml @@ -38,22 +38,16 @@ policy_templates: title: Collect logs from Nginx instances description: Collecting Nginx access and error logs - type: httpjson - title: Collect Nginx logs from third-party REST API - description: Collecting Nginx logs via third-party REST API + title: Collect logs from third-party REST API (experimental) + description: Collect logs from third-party REST API (experimental) vars: - - name: server + - name: url type: text - title: Address of Splunk Enterprise Server - description: hostname or IP + title: URL of Splunk Enterprise Server + description: i.e. scheme://host:port, path is automatic show_user: true required: true - default: server.example.com - - name: port - type: text - title: Port number of Splunk Enterprise REST API - show_user: true - required: true - default: 8089 + default: https://server.example.com:8089 - name: username type: text title: Splunk REST API Username @@ -64,28 +58,13 @@ policy_templates: title: Splunk REST API Password required: true show_user: true - - name: ssl.enabled - type: bool - title: SSL enabled + - name: ssl + type: yaml + title: SSL Configuration + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. multi: false required: false show_user: false - default: true - - name: ssl.verification_mode - type: text - title: Mode of server verification - description: "valid values: none, strict, certificate or full" - multi: false - required: false - show_user: false - default: full - - name: ssl.certificate_authorities - type: text - title: List of root certificates for TLS server verification - description: PEM encoded - multi: true - required: false - show_user: false - type: nginx/metrics vars: - name: hosts diff --git a/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs index 747247307c9..dde6c3d05a3 100644 --- a/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs index 78a56a49d64..2dae2f71b7f 100644 --- a/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs index 7ed931f7add..0a299ad717d 100644 --- a/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs index 3fd3a2a74a1..703b8dce42e 100644 --- a/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -164,4 +154,4 @@ processors: network.protocol: dhcp - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs index a4aeea075df..a560ae6be7b 100644 --- a/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs index d01b8da8f0a..fd594b57b83 100644 --- a/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs index ec1a16c8615..f4cecbbd90c 100644 --- a/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs index 6e9c749c67e..c55b89d0980 100644 --- a/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs index 8d62b91f787..5ad2f4519f0 100644 --- a/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -138,4 +128,4 @@ processors: network.protocol: ftp - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs index fde7885c046..21c78e9cd15 100644 --- a/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs index d7b9444eb05..15238519ae2 100644 --- a/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs index af7d06f90c4..3cea6b8d9f4 100644 --- a/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -129,4 +119,4 @@ processors: network.protocol: irc - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs index f8e19e2c2f5..334ef907fd6 100644 --- a/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -152,4 +142,4 @@ processors: network.protocol: kerberos - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs index b19225b810b..3a5128b0ad4 100644 --- a/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -132,4 +122,4 @@ processors: network.protocol: modbus - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs index ee5d9efc553..197baeb99c2 100644 --- a/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -132,4 +122,4 @@ processors: network.protocol: mysql - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs index 06782cc5843..affbb362e95 100644 --- a/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -149,4 +139,4 @@ processors: ecs.version: 1.7.0 - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs index 5a8920648f4..a43eef85df6 100644 --- a/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -142,4 +132,4 @@ processors: network.protocol: ntlm - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs index 22c009835f9..a362e368c26 100644 --- a/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -117,4 +107,4 @@ processors: network.transport: tcp - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs index 31cb372fd40..2b8b0f769c2 100644 --- a/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -98,4 +88,4 @@ processors: ecs.version: 1.7.0 - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs index 9ae128edeb9..10b32133fc5 100644 --- a/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -118,4 +108,4 @@ processors: netwokr.protocol: radius - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs index 640167940a9..0d4b1a7bc2c 100644 --- a/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -137,4 +127,4 @@ processors: network.protocol: rdp - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs index 4873f73e76e..ffbf005c4c2 100644 --- a/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -127,4 +117,4 @@ processors: network.protocol: rfb - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs index 2b5f1ce8806..7254b2dd8f3 100644 --- a/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs index dc1c6886775..dbd0716a22c 100644 --- a/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -151,4 +141,4 @@ processors: network.protocol: smb - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs index 17a9d8db944..6ad637412b9 100644 --- a/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -121,4 +111,4 @@ processors: network.protocol: smb - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs index 0a7b1cf61b2..194affe21b1 100644 --- a/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -117,4 +107,4 @@ processors: network.protocol: smb - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs index b2b9173ba31..8d642ae7eb9 100644 --- a/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs index e6465dcc0df..c93677a7f23 100644 --- a/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -125,4 +115,4 @@ processors: network.protocol: snmp - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs index 9be356a6775..cbb22f9b8e3 100644 --- a/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs index f9c7f36e4a2..e150630031f 100644 --- a/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs index 45b2f0a9941..23835bc1c0a 100644 --- a/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: @@ -132,4 +122,4 @@ processors: network.transport: tcp - drop_fields: fields: ["json","message"] - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs index ad308f0fccd..0837dde3f0c 100644 --- a/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs index de60140037b..232095de700 100644 --- a/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs index b95ee68a624..16a86c9c563 100644 --- a/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs index 0428351791f..fdc77560537 100644 --- a/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs index ce1c305a4f0..b847b92afd3 100644 --- a/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs index 147c052fe04..e873702051f 100644 --- a/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs @@ -5,19 +5,9 @@ auth.basic.password: {{password}} cursor: index_earliest: value: '[[.last_event.result.max_indextime]]' -{{#if ssl.enabled}} -request.url: https://{{server}}:{{port}}/services/search/jobs/export -{{#if ssl.verification_mode}} -ssl.verification_mode: {{ssl.verification_mode}} -{{/if}} -{{#if ssl.certificate_authorities}} -ssl.certificate_authorites: -{{#each ssl.certificate_authorities}} - - {{this}} -{{/each}} -{{/if}} -{{else}} -request.url: http://{{server}}:{{port}}/services/search/jobs/export +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} {{/if}} request.method: POST request.transforms: diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index c7e10151c0e..cea1b691d47 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -40,22 +40,16 @@ policy_templates: - /opt/zeek/logs/current - /usr/local/var/spool/zeek - type: httpjson - title: Collect Zeek logs from third-party REST API - description: "Collects Zeek logs from third-party REST API. Supported logs include: connection" + title: Collect logs from third-party REST API (experimental) + description: Collect logs from third-party REST API (experimental) vars: - - name: server + - name: url type: text - title: Address of Splunk Enterprise Server - description: hostname or IP + title: URL of Splunk Enterprise Server + description: i.e. scheme://host:port, path is automatic show_user: true required: true - default: server.example.com - - name: port - type: text - title: Port number of Splunk Enterprise REST API - show_user: true - required: true - default: 8089 + default: https://server.example.com:8089 - name: username type: text title: Splunk REST API Username @@ -66,27 +60,12 @@ policy_templates: title: Splunk REST API Password required: true show_user: true - - name: ssl.enabled - type: bool - title: SSL enabled + - name: ssl + type: yaml + title: SSL Configuration + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. multi: false required: false show_user: false - default: true - - name: ssl.verification_mode - type: text - title: Mode of server verification - description: "valid values: none, strict, certificate or full" - multi: false - required: false - show_user: false - default: full - - name: ssl.certificate_authorities - type: text - title: List of root certificates for TLS server verification - description: PEM encoded - multi: true - required: false - show_user: false owner: github: elastic/security-external-integrations