Add third-party REST API (elastic#545)

* Add third-party REST API - apache - nginx - zeek - aws/cloudtrail
build-security · Mar 11, 2021 · 6a5cce9 · 6a5cce9
1 parent 725cd19
commit 6a5cce9
Show file tree

Hide file tree

Showing 91 changed files with 6,202 additions and 11 deletions.
diff --git a/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs b/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs
@@ -0,0 +1,78 @@
+config_version: "2"
+interval: {{interval}}
+auth.basic.user: {{username}}
+auth.basic.password: {{password}}
+cursor:
+  index_earliest:
+    value: '[[.last_event.result.max_indextime]]'
+request.url: {{url}}/services/search/jobs/export
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.transforms:
+  - set:
+      target: url.params.search
+      value: |-
+        {{search}} | streamstats max(_indextime) AS max_indextime
+  - set:
+      target: url.params.output_mode
+      value: "json"
+  - set:
+      target: url.params.index_earliest
+      value: '[[ .cursor.index_earliest ]]'
+      default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
+  - set:
+      target: url.params.index_latest
+      value: '[[(now).Unix]]'
+  - set:
+      target: header.Content-Type
+      value: application/x-www-form-urlencoded
+response.decode_as: application/x-ndjson
+response.split:
+  target: body.result._raw
+  type: string
+  delimiter: "\n"
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - decode_json_fields:
+      fields: message
+      target: json
+      add_error_key: true
+  - drop_event:
+      when:
+        not:
+          has_fields: ['json.result']
+  - fingerprint:
+      fields:
+        - json.result._cd
+        - json.result._indextime
+        - json.result._raw
+        - json.result._time
+        - json.result.host
+        - json.result.source
+      target_field: "@metadata._id"
+  - drop_fields:
+      fields: message
+  - rename:
+      fields:
+        - from: json.result._raw
+          to: message
+        - from: json.result.host
+          to: host.name
+        - from: json.result.source
+          to: file.path
+      ignore_missing: true
+      fail_on_error: false
+  - drop_fields:
+      fields: json
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.7.0
diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -40,6 +40,10 @@ processors:
     ignore_missing: true
     patterns:
     - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
+- remove:
+    field: event.created
+    ignore_missing: true
+    ignore_failure: true
 - rename:
     field: '@timestamp'
     target_field: event.created

diff --git a/packages/apache/data_stream/access/manifest.yml b/packages/apache/data_stream/access/manifest.yml
@@ -17,3 +17,29 @@ streams:
     template_path: log.yml.hbs
     title: Apache access logs
     description: Collect Apache access logs
+  - input: httpjson
+    title: Apache access logs via Splunk Enterprise REST API
+    description: Collect apache access logs via Splunk Enterprise REST API
+    enabled: false
+    template_path: httpjson.yml.hbs
+    vars:
+      - name: interval
+        type: text
+        title: Interval to query Splunk Enterprise REST API
+        description: Go Duration syntax (eg. 10s)
+        show_user: true
+        required: true
+        default: 10s
+      - name: search
+        type: text
+        title: Splunk search string
+        show_user: true
+        required: true
+        default: "search sourcetype=\"access*\""
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+        default:
+          - forwarded
diff --git a/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs b/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs
@@ -0,0 +1,78 @@
+config_version: 2
+interval: {{interval}}
+auth.basic.user: {{username}}
+auth.basic.password: {{password}}
+cursor:
+  index_earliest:
+    value: '[[.last_event.result.max_indextime]]'
+request.url: {{url}}/services/search/jobs/export
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.transforms:
+  - set:
+      target: url.params.search
+      value: |-
+        {{search}} | streamstats max(_indextime) AS max_indextime
+  - set:
+      target: url.params.output_mode
+      value: "json"
+  - set:
+      target: url.params.index_earliest
+      value: '[[ .cursor.index_earliest ]]'
+      default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
+  - set:
+      target: url.params.index_latest
+      value: '[[(now).Unix]]'
+  - set:
+      target: header.Content-Type
+      value: application/x-www-form-urlencoded
+response.decode_as: application/x-ndjson
+response.split:
+  target: body.result._raw
+  type: string
+  delimiter: "\n"
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - decode_json_fields:
+      fields: message
+      target: json
+      add_error_key: true
+  - drop_event:
+      when:
+        not:
+          has_fields: ['json.result']
+  - fingerprint:
+      fields:
+        - json.result._cd
+        - json.result._indextime
+        - json.result._raw
+        - json.result._time
+        - json.result.host
+        - json.result.source
+      target_field: "@metadata._id"
+  - drop_fields:
+      fields: message
+  - rename:
+      fields:
+        - from: json.result._raw
+          to: message
+        - from: json.result.host
+          to: host.name
+        - from: json.result.source
+          to: file.path
+      ignore_missing: true
+      fail_on_error: false
+  - drop_fields:
+      fields: json
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.7.0
diff --git a/packages/apache/data_stream/error/manifest.yml b/packages/apache/data_stream/error/manifest.yml
@@ -16,3 +16,29 @@ streams:
     template_path: log.yml.hbs
     title: Apache error logs
     description: Collect Apache error logs
+  - input: httpjson
+    title: Apache error logs via Splunk Enterprise REST API
+    description: Collect apache error logs via Splunk Enterprise REST API
+    enabled: false
+    template_path: httpjson.yml.hbs
+    vars:
+      - name: interval
+        type: text
+        title: Interval to query Splunk Enterprise REST API
+        description: Go Duration syntax (eg. 10s)
+        show_user: true
+        required: true
+        default: 10s
+      - name: search
+        type: text
+        title: Splunk search string
+        show_user: true
+        required: true
+        default: search sourcetype=apache:error OR sourcetype=apache_error
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+        default:
+          - forwarded
diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml
@@ -1,15 +1,15 @@
 format_version: 1.0.0
 name: apache
 title: Apache
-version: 0.3.4
+version: 0.4.0
 license: basic
 description: Apache Integration
 type: integration
 categories:
   - web
 release: experimental
 conditions:
-  kibana.version: '^7.11.0'
+  kibana.version: '^7.12.0'
 screenshots:
   - src: /img/apache-metrics-overview.png
     title: Apache metrics overview
@@ -32,6 +32,34 @@ policy_templates:
       - type: logfile
         title: Collect logs from Apache instances
         description: Collecting Apache access and error logs
+      - type: httpjson
+        title: Collect logs from third-party REST API (experimental)
+        description: Collect logs from third-party REST API (experimental)
+        vars:
+          - name: url
+            type: text
+            title: URL of Splunk Enterprise Server
+            description: i.e. scheme://host:port,  path is automatic
+            show_user: true
+            required: true
+            default: https://server.example.com:8089
+          - name: username
+            type: text
+            title: Splunk REST API Username
+            show_user: true
+            required: true
+          - name: password
+            type: password
+            title: Splunk REST API Password
+            required: true
+            show_user: true
+          - name: ssl
+            type: yaml
+            title: SSL Configuration
+            description: i.e. certificate_authorities, supported_protocols, verification_mode etc.
+            multi: false
+            required: false
+            show_user: false
       - type: apache/metrics
         title: Collect metrics from Apache instances
         description: Collecting Apache status metrics

diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs
@@ -0,0 +1,70 @@
+config_version: 2
+interval: {{interval}}
+auth.basic.user: {{username}}
+auth.basic.password: {{password}}
+cursor:
+  index_earliest:
+    value: '[[.last_event.result.max_indextime]]'
+request.url: {{url}}/services/search/jobs/export
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.transforms:
+  - set:
+      target: url.params.search
+      value: {{search}} | streamstats max(_indextime) AS max_indextime
+  - set:
+      target: url.params.output_mode
+      value: "json"
+  - set:
+      target: url.params.index_earliest
+      value: '[[ .cursor.index_earliest ]]'
+      default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
+  - set:
+      target: url.params.index_latest
+      value: '[[(now).Unix]]'
+  - set:
+      target: header.Content-Type
+      value: application/x-www-form-urlencoded
+response.decode_as: application/x-ndjson
+response.split:
+  target: body.result._raw
+  type: string
+  delimiter: "\n"
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+processors:
+  - decode_json_fields:
+      fields: message
+      target: json
+      add_error_key: true
+  - drop_event:
+      when:
+        not:
+          has_fields: ['json.result']
+  - fingerprint:
+      fields:
+        - json.result._cd
+        - json.result._indextime
+        - json.result._raw
+        - json.result._time
+      target_field: "@metadata._id"
+  - drop_fields:
+      fields: ["message"]
+  - rename:
+      fields:
+        - from: json.result._raw
+          to: message
+  - drop_fields:
+      fields: ["json"]
+      ignore_missing: true
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.8.0
diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml
@@ -22,3 +22,29 @@ streams:
         required: false
         show_user: false
         description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.
+  - input: httpjson
+    title: AWS CloudTrail logs via Splunk Enterprise REST API
+    description: Collect AWS CloudTrail logs via Splunk Enterprise REST API
+    enabled: false
+    template_path: httpjson.yml.hbs
+    vars:
+      - name: interval
+        type: text
+        title: Interval to query Splunk Enterprise REST API
+        description: Go Duration syntax (eg. 10s)
+        show_user: true
+        required: true
+        default: 10s
+      - name: search
+        type: text
+        title: Splunk search string
+        show_user: true
+        required: true
+        default: "search sourcetype=aws:cloudtrail"
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+        default:
+          - forwarded