diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e5e93a3a9eb..a8d4379a450 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -746,6 +746,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve Nats filebeat dashboard. {pull}22726[22726] - Add support for UNIX datagram sockets in `unix` input. {issues}18632[18632] {pull}22699[22699] - Add new httpjson input features and mark old config ones for deprecation {pull}22320[22320] +- Add support for Snyk Vulnerability and Audit API. {pull}22677[22677] - Add logic for external network.direction in sophos xg fileset {pull}22973[22973] - Add `http.request.mime_type` for Elasticsearch audit log fileset. {pull}22975[22975] - Add configuration option to set external and internal networks for panw panos fileset {pull}22998[22998] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 74d6716e405..18da50aa354 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -78,6 +78,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -126376,6 +126377,364 @@ type: keyword -- This key captures values or decorators used within a registry entry +type: keyword + +-- + +[[exported-fields-snyk]] +== Snyk fields + +Snyk module + + + +[float] +=== snyk + +Module for parsing Snyk project vulnerabilities. + + + +*`snyk.projects`*:: ++ +-- +Array with all related projects objects. + + +type: flattened + +-- + +*`snyk.related.projects`*:: ++ +-- +Array of all the related project ID's. + + +type: keyword + +-- + +[float] +=== audit + +Module for parsing Snyk audit logs. + + + +*`snyk.audit.org_id`*:: ++ +-- +ID of the related Organization related to the event. + + +type: keyword + +-- + +*`snyk.audit.project_id`*:: ++ +-- +ID of the project related to the event. + + +type: keyword + +-- + +*`snyk.audit.content`*:: ++ +-- +Overview of the content that was changed, both old and new values. + + +type: flattened + +-- + +[float] +=== vulnerabilities + +Module for parsing Snyk project vulnerabilities. + + + +*`snyk.vulnerabilities.cvss3`*:: ++ +-- +CSSv3 scores. + + +type: keyword + +-- + +*`snyk.vulnerabilities.disclosure_time`*:: ++ +-- +The time this vulnerability was originally disclosed to the package maintainers. + + +type: date + +-- + +*`snyk.vulnerabilities.exploit_maturity`*:: ++ +-- +The Snyk exploit maturity level. + + +type: keyword + +-- + +*`snyk.vulnerabilities.id`*:: ++ +-- +The vulnerability reference ID. + + +type: keyword + +-- + +*`snyk.vulnerabilities.is_ignored`*:: ++ +-- +If the vulnerability report has been ignored. + + +type: boolean + +-- + +*`snyk.vulnerabilities.is_patchable`*:: ++ +-- +If vulnerability is fixable by using a Snyk supplied patch. + + +type: boolean + +-- + +*`snyk.vulnerabilities.is_patched`*:: ++ +-- +If the vulnerability has been patched. + + +type: boolean + +-- + +*`snyk.vulnerabilities.is_pinnable`*:: ++ +-- +If the vulnerability is fixable by pinning a transitive dependency. + + +type: boolean + +-- + +*`snyk.vulnerabilities.is_upgradable`*:: ++ +-- +If the vulnerability fixable by upgrading a dependency. + + +type: boolean + +-- + +*`snyk.vulnerabilities.language`*:: ++ +-- +The package's programming language. + + +type: keyword + +-- + +*`snyk.vulnerabilities.package`*:: ++ +-- +The package identifier according to its package manager. + + +type: keyword + +-- + +*`snyk.vulnerabilities.package_manager`*:: ++ +-- +The package manager. + + +type: keyword + +-- + +*`snyk.vulnerabilities.patches`*:: ++ +-- +Patches required to resolve the issue created by Snyk. + + +type: flattened + +-- + +*`snyk.vulnerabilities.priority_score`*:: ++ +-- +The CVS priority score. + + +type: long + +-- + +*`snyk.vulnerabilities.publication_time`*:: ++ +-- +The vulnerability publication time. + + +type: date + +-- + +*`snyk.vulnerabilities.jira_issue_url`*:: ++ +-- +Link to the related Jira issue. + + +type: keyword + +-- + +*`snyk.vulnerabilities.original_severity`*:: ++ +-- +The original severity of the vulnerability. + + +type: long + +-- + +*`snyk.vulnerabilities.reachability`*:: ++ +-- +If the vulnerable function from the library is used in the code scanned. Can either be No Info, Potentially reachable and Reachable. + + +type: keyword + +-- + +*`snyk.vulnerabilities.title`*:: ++ +-- +The issue title. + + +type: keyword + +-- + +*`snyk.vulnerabilities.type`*:: ++ +-- +The issue type. Can be either "license" or "vulnerability". + + +type: keyword + +-- + +*`snyk.vulnerabilities.unique_severities_list`*:: ++ +-- +A list of related unique severities. + + +type: keyword + +-- + +*`snyk.vulnerabilities.version`*:: ++ +-- +The package version this issue is applicable to. + + +type: keyword + +-- + +*`snyk.vulnerabilities.introduced_date`*:: ++ +-- +The date the vulnerability was initially found. + + +type: date + +-- + +*`snyk.vulnerabilities.is_fixed`*:: ++ +-- +If the related vulnerability has been resolved. + + +type: boolean + +-- + +*`snyk.vulnerabilities.credit`*:: ++ +-- +Reference to the person that original found the vulnerability. + + +type: keyword + +-- + +*`snyk.vulnerabilities.semver`*:: ++ +-- +One or more semver ranges this issue is applicable to. The format varies according to package manager. + + +type: flattened + +-- + +*`snyk.vulnerabilities.identifiers.alternative`*:: ++ +-- +Additional vulnerability identifiers. + + +type: keyword + +-- + +*`snyk.vulnerabilities.identifiers.cwe`*:: ++ +-- +CWE vulnerability identifiers. + + type: keyword -- diff --git a/filebeat/docs/modules/snyk.asciidoc b/filebeat/docs/modules/snyk.asciidoc new file mode 100644 index 00000000000..68e6c291fb1 --- /dev/null +++ b/filebeat/docs/modules/snyk.asciidoc @@ -0,0 +1,244 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-snyk]] +[role="xpack"] + +:modulename: snyk +:has-dashboards: false + + +== Snyk module +beta[] + +This is a module for ingesting data from the different Snyk API Endpoints. Currently supports these filesets: + +- `vulnerabilities` fileset: Collects all found vulnerabilities for the related organizations and projects +- `audit` fileset: Collects audit logging from Snyk, this can be actions like users, permissions, groups, api access and more. + +include::../include/what-happens.asciidoc[] + +include::../include/gs-link.asciidoc[] + +include::../include/configuring-intro.asciidoc[] + +:fileset_ex: audit + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `audit` fileset settings + +beta[] + +To configure access for Filebeat to the Snyk Audit Log API you will have to generate an API access token as described in the https://snyk.docs.apiary.io/#introduction/authorization[Snyk Documentation] + +Example config: + +[source,yaml] +---- +- module: snyk + audit: + var.input: httpjson + var.audit_type: organization + var.audit_id: 1235432-asdfdf-2341234-asdgjhg + var.interval: 1h + var.api_token: 53453Sddf8-7fsf-414234gfd-9sdfb7-5asdfh9f8e342 +---- + +There is also multiple optional configuration options that can be used to filter out unwanted content, an example below: + +[source,yaml] +---- +- module: snyk + audit: + var.input: httpjson + var.audit_type: organization + var.audit_id: 1235432-asdfdf-2341234-asdgjhg + var.interval: 1h + var.api_token: 53453Sddf8-7fsf-414234gfd-9sdfb7-5asdfh9f8e342 + var.email_address: "test@example.com" +---- + +include::../include/var-paths.asciidoc[] + +*`var.first_interval`*:: + +How far to look back the first time the module starts, this supports values in full days (24h, 48h etc). + +*`var.audit_type`*:: + +What audit type to collect, can be either "group" or "organization". + +*`var.audit_id`*:: + +The ID related to the audit_type. If audit type is group, then this value should be the group ID, or if it is organization it should be the organization ID to collect from. + +*`var.api_token`*:: + +The API token that is created for a specific user, found in the Snyk management dashboard. + +*`var.project_id`*:: + +Optional field for filtering, will return only logs for this specific project. + +*`var.user_id`*:: + +Optional field for filtering, user public ID. Will fetch only audit logs originated from this user's actions. + +*`var.event`*:: + +Optional field for filtering, will return only logs for this specific event. + +*`var.email_address`*:: + +Optional field for filtering, User email address. Will fetch only audit logs originated from this user's actions. + +[float] +==== Snyk Audit Log ECS Fields + +This is a list of Snyk Audit Log fields that are mapped to ECS. + +[options="header"] +|====================================================================== +| Snyk Audit log fields | ECS Fields | +| groupId | user.group.id | +| userId | user.id | +| event | event.action | +| created | @timestamp | +|====================================================================== + +:fileset_ex: vulnerabilities + +[float] +==== `vulnerabilities` fileset settings + +beta[] + +To configure access for Filebeat to the Snyk Vulnerabilities API you will have to generate an API access token as described in the https://snyk.docs.apiary.io/#introduction/authorization[Snyk Documentation] + +Example config: + +[source,yaml] +---- +- module: snyk + vulnerabilities: + var.input: httpjson + var.interval: 24h + var.api_token: 53453Sddf8-7fsf-414234gfd-9sdfb7-5asdfh9f8e342 + var.orgs: + - 12354-asdfdf-123543-asdsdfg + - 76554-jhggfd-654342-hgrfasd +---- + +There is also multiple optional configuration options that can be used to filter out unwanted content, an example below: + +[source,yaml] +---- +- module: snyk + vulnerabilities: + var.input: httpjson + var.interval: 24h + var.api_token: 53453Sddf8-7fsf-414234gfd-9sdfb7-5asdfh9f8e342 + var.orgs: + - 12354-asdfdf-123543-asdsdfg + - 76554-jhggfd-654342-hgrfasd + var.included_severity: + - medium + - high + var.types: + - vuln +---- + +include::../include/var-paths.asciidoc[] + +*`var.first_interval`*:: + +How far to look back the first time the module starts, this supports values in full days (24h, 48h etc). + +*`var.api_token`*:: + +The API token that is created for a specific user, found in the Snyk management dashboard. + +*`var.orgs`*:: + +The list of org IDs to filter the results by. One organization ID per line, starting with a - sign + +*`var.included_severity`*:: + +Optional list of fields for filtering, the severity levels of issues to filter the results by. + +*`var.exploit_maturit`*:: + +Optional list of fields for filtering, the exploit maturity levels of issues to filter the results by. + +*`var.types`*:: + +Optional list of fields for filtering, the type of issues to filter the results by. + +*`var.languages`*:: + +Optional list of fields for filtering, the type of languages to filter the results by. + +*`var.identifier`*:: + +Optional field for filtering, search term to filter issue name by, or an exact CVE or CWE. + +*`var.ignored`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.patched`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.fixable`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.is_fixed`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.is_patchable`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.is_pinnable`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.min_priority_score`*:: + +Optional field for filtering, The minimum priority score ranging between 0-1000 + +*`var.max_priority_score`*:: + +Optional field for filtering, The maximum priority score ranging between 0-1000 + +[float] +==== Snyk Audit Log ECS Fields + +This is a list of Snyk Vulnerability fields that are mapped to ECS. + +[options="header"] +|============================================================| +| Snyk Fields | ECS Fields | +| issue.description | vulnerability.description | +| issue.identifiers.CVE | vulnerability.id | +| issue.identifiers.ALTERNATIVE | vulnerability.id | +| issue.cvssScore | vulnerability.score.base | +| issue.severity | vulnerability.severity | +| issue.url | vulnerability.reference | +|============================================================| + +:modulename!: + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index b0b123a5406..d4691215e37 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -57,6 +57,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -127,6 +128,7 @@ include::modules/radware.asciidoc[] include::modules/redis.asciidoc[] include::modules/santa.asciidoc[] include::modules/snort.asciidoc[] +include::modules/snyk.asciidoc[] include::modules/sonicwall.asciidoc[] include::modules/sophos.asciidoc[] include::modules/squid.asciidoc[] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index fa3caa93475..53715c59d3d 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -269,6 +269,7 @@ def clean_keys(obj): "gsuite.saml", "gsuite.user_accounts", "zoom.webhook", + "snyk.vulnerabilities", } # dataset + log file pairs for which @timestamp is kept as an exception from above remove_timestamp_exception = { diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 840f79bf238..cd2d2d9dfd4 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1707,6 +1707,117 @@ filebeat.modules: # "+02:00" for GMT+02:00 # var.tz_offset: local +#--------------------------------- Snyk Module --------------------------------- +- module: snyk + audit: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + # + # What audit type to collect, can be either "group" or "organization". + #var.audit_type: organization + # + # The ID related to the audit_type. If audit type is group, then this value should be + # the group ID and if it is organization it should be the organization ID to collect from. + #var.audit_id: 1235432-asdfdf-2341234-asdgjhg + + # How often the API should be polled, defaults to 1 hour. + #var.interval: 1h + # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). + #var.first_interval: 24h + + # The API token that is created for a specific user, found in the Snyk management dashboard. + #var.api_token: + + # Event filtering. + # All configuration items below is OPTIONAL and the default options will be overwritten + # for each entry that is not commented out. + + # Will return only logs for this specific project. + #var.project_id: "" + # User public ID. Will fetch only audit logs originated from this user's actions. + #var.user_id: "" + # Will return only logs for this specific event. + #var.event: "" + # User email address. Will fetch only audit logs originated from this user's actions. + #var.email_address: "" + + vulnerabilities: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + + # How often the API should be polled. Data from the Snyk API is automatically updated + # once per day, so the default interval is 24 hours. + #var.interval: 24h + + # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). + #var.first_interval: 24h + + # The API token that is created for a specific user, found in the Snyk management dashboard. + #var.api_token: + + # The list of org IDs to filter the results by. + # One organization ID per line, starting with a - sign + #var.orgs: + # - 12354-asdfdf-123543-asdsdfg + # - 76554-jhggfd-654342-hgrfasd + + + # Event filtering. + # All configuration items below is OPTIONAL and the default options will be overwritten + # for each entry that is not commented out. + + # The severity levels of issues to filter the results by. + #var.included_severity: + # - high + # - medium + # - low + # + # The exploit maturity levels of issues to filter the results by. + #var.exploit_maturity: + # - mature + # - proof-of-concept + # - no-known-exploit + # - no-data + # + # The type of issues to filter the results by. + #var.types: + # - vuln + # - license + # + # The type of languages to filter the results by. + #var.languages: + # - javascript + # - ruby + # - java + # - scala + # - python + # - golang + # - php + # - dotnet + # - swift + # - docker + # + # Search term to filter issue name by, or an exact CVE or CWE. + #var.identifier: + # - "" + # + # If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + #var.ignored: false + #var.patched: false + #var.fixable: false + #var.is_fixed: false + #var.is_patchable: false + #var.is_pinnable: false + # + # The priority score ranging between 0-1000 + #var.min_priority_score: 0 + #var.max_priority_score: 1000 + + #----------------------------- Sonicwall-FW Module ----------------------------- - module: sonicwall firewall: diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 086e96d2de2..80824451de4 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -50,6 +50,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/module/rabbitmq" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/radware" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/snort" + _ "github.com/elastic/beats/v7/x-pack/filebeat/module/snyk" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/sonicwall" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/sophos" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/squid" diff --git a/x-pack/filebeat/module/snyk/_meta/config.yml b/x-pack/filebeat/module/snyk/_meta/config.yml new file mode 100644 index 00000000000..d3e1b4da23a --- /dev/null +++ b/x-pack/filebeat/module/snyk/_meta/config.yml @@ -0,0 +1,109 @@ +- module: snyk + audit: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + # + # What audit type to collect, can be either "group" or "organization". + #var.audit_type: organization + # + # The ID related to the audit_type. If audit type is group, then this value should be + # the group ID and if it is organization it should be the organization ID to collect from. + #var.audit_id: 1235432-asdfdf-2341234-asdgjhg + + # How often the API should be polled, defaults to 1 hour. + #var.interval: 1h + # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). + #var.first_interval: 24h + + # The API token that is created for a specific user, found in the Snyk management dashboard. + #var.api_token: + + # Event filtering. + # All configuration items below is OPTIONAL and the default options will be overwritten + # for each entry that is not commented out. + + # Will return only logs for this specific project. + #var.project_id: "" + # User public ID. Will fetch only audit logs originated from this user's actions. + #var.user_id: "" + # Will return only logs for this specific event. + #var.event: "" + # User email address. Will fetch only audit logs originated from this user's actions. + #var.email_address: "" + + vulnerabilities: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + + # How often the API should be polled. Data from the Snyk API is automatically updated + # once per day, so the default interval is 24 hours. + #var.interval: 24h + + # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). + #var.first_interval: 24h + + # The API token that is created for a specific user, found in the Snyk management dashboard. + #var.api_token: + + # The list of org IDs to filter the results by. + # One organization ID per line, starting with a - sign + #var.orgs: + # - 12354-asdfdf-123543-asdsdfg + # - 76554-jhggfd-654342-hgrfasd + + + # Event filtering. + # All configuration items below is OPTIONAL and the default options will be overwritten + # for each entry that is not commented out. + + # The severity levels of issues to filter the results by. + #var.included_severity: + # - high + # - medium + # - low + # + # The exploit maturity levels of issues to filter the results by. + #var.exploit_maturity: + # - mature + # - proof-of-concept + # - no-known-exploit + # - no-data + # + # The type of issues to filter the results by. + #var.types: + # - vuln + # - license + # + # The type of languages to filter the results by. + #var.languages: + # - javascript + # - ruby + # - java + # - scala + # - python + # - golang + # - php + # - dotnet + # - swift + # - docker + # + # Search term to filter issue name by, or an exact CVE or CWE. + #var.identifier: + # - "" + # + # If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + #var.ignored: false + #var.patched: false + #var.fixable: false + #var.is_fixed: false + #var.is_patchable: false + #var.is_pinnable: false + # + # The priority score ranging between 0-1000 + #var.min_priority_score: 0 + #var.max_priority_score: 1000 + diff --git a/x-pack/filebeat/module/snyk/_meta/docs.asciidoc b/x-pack/filebeat/module/snyk/_meta/docs.asciidoc new file mode 100644 index 00000000000..7781e5bdb50 --- /dev/null +++ b/x-pack/filebeat/module/snyk/_meta/docs.asciidoc @@ -0,0 +1,231 @@ +[role="xpack"] + +:modulename: snyk +:has-dashboards: false + + +== Snyk module +beta[] + +This is a module for ingesting data from the different Snyk API Endpoints. Currently supports these filesets: + +- `vulnerabilities` fileset: Collects all found vulnerabilities for the related organizations and projects +- `audit` fileset: Collects audit logging from Snyk, this can be actions like users, permissions, groups, api access and more. + +include::../include/what-happens.asciidoc[] + +include::../include/gs-link.asciidoc[] + +include::../include/configuring-intro.asciidoc[] + +:fileset_ex: audit + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `audit` fileset settings + +beta[] + +To configure access for Filebeat to the Snyk Audit Log API you will have to generate an API access token as described in the https://snyk.docs.apiary.io/#introduction/authorization[Snyk Documentation] + +Example config: + +[source,yaml] +---- +- module: snyk + audit: + var.input: httpjson + var.audit_type: organization + var.audit_id: 1235432-asdfdf-2341234-asdgjhg + var.interval: 1h + var.api_token: 53453Sddf8-7fsf-414234gfd-9sdfb7-5asdfh9f8e342 +---- + +There is also multiple optional configuration options that can be used to filter out unwanted content, an example below: + +[source,yaml] +---- +- module: snyk + audit: + var.input: httpjson + var.audit_type: organization + var.audit_id: 1235432-asdfdf-2341234-asdgjhg + var.interval: 1h + var.api_token: 53453Sddf8-7fsf-414234gfd-9sdfb7-5asdfh9f8e342 + var.email_address: "test@example.com" +---- + +include::../include/var-paths.asciidoc[] + +*`var.first_interval`*:: + +How far to look back the first time the module starts, this supports values in full days (24h, 48h etc). + +*`var.audit_type`*:: + +What audit type to collect, can be either "group" or "organization". + +*`var.audit_id`*:: + +The ID related to the audit_type. If audit type is group, then this value should be the group ID, or if it is organization it should be the organization ID to collect from. + +*`var.api_token`*:: + +The API token that is created for a specific user, found in the Snyk management dashboard. + +*`var.project_id`*:: + +Optional field for filtering, will return only logs for this specific project. + +*`var.user_id`*:: + +Optional field for filtering, user public ID. Will fetch only audit logs originated from this user's actions. + +*`var.event`*:: + +Optional field for filtering, will return only logs for this specific event. + +*`var.email_address`*:: + +Optional field for filtering, User email address. Will fetch only audit logs originated from this user's actions. + +[float] +==== Snyk Audit Log ECS Fields + +This is a list of Snyk Audit Log fields that are mapped to ECS. + +[options="header"] +|====================================================================== +| Snyk Audit log fields | ECS Fields | +| groupId | user.group.id | +| userId | user.id | +| event | event.action | +| created | @timestamp | +|====================================================================== + +:fileset_ex: vulnerabilities + +[float] +==== `vulnerabilities` fileset settings + +beta[] + +To configure access for Filebeat to the Snyk Vulnerabilities API you will have to generate an API access token as described in the https://snyk.docs.apiary.io/#introduction/authorization[Snyk Documentation] + +Example config: + +[source,yaml] +---- +- module: snyk + vulnerabilities: + var.input: httpjson + var.interval: 24h + var.api_token: 53453Sddf8-7fsf-414234gfd-9sdfb7-5asdfh9f8e342 + var.orgs: + - 12354-asdfdf-123543-asdsdfg + - 76554-jhggfd-654342-hgrfasd +---- + +There is also multiple optional configuration options that can be used to filter out unwanted content, an example below: + +[source,yaml] +---- +- module: snyk + vulnerabilities: + var.input: httpjson + var.interval: 24h + var.api_token: 53453Sddf8-7fsf-414234gfd-9sdfb7-5asdfh9f8e342 + var.orgs: + - 12354-asdfdf-123543-asdsdfg + - 76554-jhggfd-654342-hgrfasd + var.included_severity: + - medium + - high + var.types: + - vuln +---- + +include::../include/var-paths.asciidoc[] + +*`var.first_interval`*:: + +How far to look back the first time the module starts, this supports values in full days (24h, 48h etc). + +*`var.api_token`*:: + +The API token that is created for a specific user, found in the Snyk management dashboard. + +*`var.orgs`*:: + +The list of org IDs to filter the results by. One organization ID per line, starting with a - sign + +*`var.included_severity`*:: + +Optional list of fields for filtering, the severity levels of issues to filter the results by. + +*`var.exploit_maturit`*:: + +Optional list of fields for filtering, the exploit maturity levels of issues to filter the results by. + +*`var.types`*:: + +Optional list of fields for filtering, the type of issues to filter the results by. + +*`var.languages`*:: + +Optional list of fields for filtering, the type of languages to filter the results by. + +*`var.identifier`*:: + +Optional field for filtering, search term to filter issue name by, or an exact CVE or CWE. + +*`var.ignored`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.patched`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.fixable`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.is_fixed`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.is_patchable`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.is_pinnable`*:: + +Optional field for filtering, If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + +*`var.min_priority_score`*:: + +Optional field for filtering, The minimum priority score ranging between 0-1000 + +*`var.max_priority_score`*:: + +Optional field for filtering, The maximum priority score ranging between 0-1000 + +[float] +==== Snyk Audit Log ECS Fields + +This is a list of Snyk Vulnerability fields that are mapped to ECS. + +[options="header"] +|============================================================| +| Snyk Fields | ECS Fields | +| issue.description | vulnerability.description | +| issue.identifiers.CVE | vulnerability.id | +| issue.identifiers.ALTERNATIVE | vulnerability.id | +| issue.cvssScore | vulnerability.score.base | +| issue.severity | vulnerability.severity | +| issue.url | vulnerability.reference | +|============================================================| + +:modulename!: diff --git a/x-pack/filebeat/module/snyk/_meta/fields.yml b/x-pack/filebeat/module/snyk/_meta/fields.yml new file mode 100644 index 00000000000..9e0796c7855 --- /dev/null +++ b/x-pack/filebeat/module/snyk/_meta/fields.yml @@ -0,0 +1,20 @@ +- key: snyk + title: Snyk + description: > + Snyk module + fields: + - name: snyk + type: group + release: beta + description: > + Module for parsing Snyk project vulnerabilities. + fields: + - name: projects + type: flattened + description: > + Array with all related projects objects. + - name: related.projects + type: keyword + description: > + Array of all the related project ID's. + diff --git a/x-pack/filebeat/module/snyk/audit/_meta/fields.yml b/x-pack/filebeat/module/snyk/audit/_meta/fields.yml new file mode 100644 index 00000000000..aea5500d4fb --- /dev/null +++ b/x-pack/filebeat/module/snyk/audit/_meta/fields.yml @@ -0,0 +1,18 @@ +- name: audit + type: group + release: beta + description: > + Module for parsing Snyk audit logs. + fields: + - name: org_id + type: keyword + description: > + ID of the related Organization related to the event. + - name: project_id + type: keyword + description: > + ID of the project related to the event. + - name: content + type: flattened + description: > + Overview of the content that was changed, both old and new values. diff --git a/x-pack/filebeat/module/snyk/audit/config/config.yml b/x-pack/filebeat/module/snyk/audit/config/config.yml new file mode 100644 index 00000000000..3a41b508690 --- /dev/null +++ b/x-pack/filebeat/module/snyk/audit/config/config.yml @@ -0,0 +1,76 @@ +{{ if eq .input "httpjson" }} + +type: httpjson +config_version: "2" +interval: {{ .interval }} +{{ if .audit_type }} +{{ if eq .audit_type "group" }} +request.url: https://snyk.io/api/v1/group/{{.audit_id}}/audit?page=1&sortOrder=ASC +{{ end }} +{{ if eq .audit_type "organization" }} +request.url: https://snyk.io/api/v1/org/{{.audit_id}}/audit?page=1&sortOrder=ASC +{{ end }} +{{ end }} +request.method: POST +request.ssl: {{ .ssl | tojson }} +request.transforms: +- set: + target: header.Authorization + value: token {{ .api_token }} +- set: + target: url.params.to + value: '[[ formatDate (now) "2006-01-02" ]]' +- set: + target: url.params.from + value: '[[ formatDate (now (parseDuration .cursor.interval)) "2006-01-02" ]]' + default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "2006-01-02" ]]' +cursor: + interval: + value: "-24h" + +request.timeout: 120s +request.body: + filters: + userId: {{ .user_id }} + email: {{ .email_address }} + event: {{ .event }} + projectId: {{ .project_id }} + +response.pagination: +- set: + target: url.value + value: '[[ getRFC5988Link "next" .last_response.header.Link ]]' + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_locale: ~ + - decode_json_fields: + fields: [message] + target: json + - fingerprint: + fields: ["json.orgId", "json.created", "json.event"] + target_field: "@metadata._id" + - script: + lang: javascript + id: my_filter + source: > + function process(event) { + event.Put("@metadata.op_type", "index"); + } + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/snyk/audit/ingest/pipeline.yml b/x-pack/filebeat/module/snyk/audit/ingest/pipeline.yml new file mode 100644 index 00000000000..00b95408b3b --- /dev/null +++ b/x-pack/filebeat/module/snyk/audit/ingest/pipeline.yml @@ -0,0 +1,67 @@ +description: Pipeline for Snyk Audit logs +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- script: + lang: painless + if: ctx?.json != null + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); +- rename: + field: json + target_field: snyk.audit + ignore_missing: true +- rename: + field: snyk.audit.groupId + target_field: user.group.id + ignore_missing: true +- rename: + field: snyk.audit.orgId + target_field: snyk.audit.org_id + ignore_missing: true +- rename: + field: snyk.audit.projectId + target_field: snyk.audit.project_id + ignore_missing: true +- rename: + field: snyk.audit.userId + target_field: user.id + ignore_missing: true +- rename: + field: snyk.audit.event + target_field: event.action + ignore_missing: true +- date: + field: snyk.audit.created + target_field: "@timestamp" + formats: + - "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" +- remove: + field: + - snyk.audit.created + - message + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/snyk/audit/manifest.yml b/x-pack/filebeat/module/snyk/audit/manifest.yml new file mode 100644 index 00000000000..eed5f4c29c9 --- /dev/null +++ b/x-pack/filebeat/module/snyk/audit/manifest.yml @@ -0,0 +1,27 @@ +module_version: 1.0 + +var: + - name: tags + default: [snyk-audit, forwarded] + - name: input + default: httpjson + - name: interval + default: 1h + - name: first_interval + default: 24h + - name: api_token + - name: audit_type + - name: audit_id + - name: user_id + default: "" + - name: project_id + default: "" + - name: email_address + default: "" + - name: event + default: "" + - name: ssl + +ingest_pipeline: + - ingest/pipeline.yml +input: config/config.yml diff --git a/x-pack/filebeat/module/snyk/audit/test/snyk_audit.ndjson.log b/x-pack/filebeat/module/snyk/audit/test/snyk_audit.ndjson.log new file mode 100644 index 00000000000..2f806ad8e78 --- /dev/null +++ b/x-pack/filebeat/module/snyk/audit/test/snyk_audit.ndjson.log @@ -0,0 +1,4 @@ +{"groupId":"groupid123test-543123-54312sadf-123ad","orgId":"orgid123test-5643asd234-asdfasdf","userId":"userid123test-234sdfa2-423sdfa-2134","projectId":null,"event":"user.logged_in","content":{"sessionPublicId":"sessionId123-t34123-sdfa234-asd"},"created":"2020-11-17T14:30:13.800Z"} +{"groupId":"groupid123test-543123-54312sadf-123ad","orgId":"orgid123test-5643asd234-asdfasdf","userId":"userid123test-234sdfa2-423sdfa-2134","projectId":null,"event":"api.access","content":{"url":"/api/v1/org/orgid123test-5643asd234-asdfasdf/projects"},"created":"2020-11-12T13:24:40.317Z"} +{"groupId":"groupid123test-543123-54312sadf-123ad","orgId":"orgid123test-5643asd234-asdfasdf","userId":"userid123test-234sdfa2-423sdfa-2134","projectId":null,"event":"org.user.invite","content":{"email":"someone@snyk.io","isAdmin":false},"created":"2020-11-11T21:00:00.000Z"} +{"groupId":"groupid123test-543123-54312sadf-123ad","orgId":"orgid123test-5643asd234-asdfasdf","userId":"userid123test-234sdfa2-423sdfa-2134","projectId":null,"event":"org.user.role.edit","content":{"userPublicId":"userid123test-234sdfa2-423sdfa-2134","before":"COLLABORATOR","after":"ADMIN"},"created":"2020-11-15T06:02:45.497Z"} diff --git a/x-pack/filebeat/module/snyk/audit/test/snyk_audit.ndjson.log-expected.json b/x-pack/filebeat/module/snyk/audit/test/snyk_audit.ndjson.log-expected.json new file mode 100644 index 00000000000..b1b637d762f --- /dev/null +++ b/x-pack/filebeat/module/snyk/audit/test/snyk_audit.ndjson.log-expected.json @@ -0,0 +1,81 @@ +[ + { + "@timestamp": "2020-11-17T14:30:13.800Z", + "event.action": "user.logged_in", + "event.dataset": "snyk.audit", + "event.module": "snyk", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 0, + "service.type": "snyk", + "snyk.audit.content.sessionPublicId": "sessionId123-t34123-sdfa234-asd", + "snyk.audit.org_id": "orgid123test-5643asd234-asdfasdf", + "tags": [ + "snyk-audit", + "forwarded" + ], + "user.group.id": "groupid123test-543123-54312sadf-123ad", + "user.id": "userid123test-234sdfa2-423sdfa-2134" + }, + { + "@timestamp": "2020-11-12T13:24:40.317Z", + "event.action": "api.access", + "event.dataset": "snyk.audit", + "event.module": "snyk", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 285, + "service.type": "snyk", + "snyk.audit.content.url": "/api/v1/org/orgid123test-5643asd234-asdfasdf/projects", + "snyk.audit.org_id": "orgid123test-5643asd234-asdfasdf", + "tags": [ + "snyk-audit", + "forwarded" + ], + "user.group.id": "groupid123test-543123-54312sadf-123ad", + "user.id": "userid123test-234sdfa2-423sdfa-2134" + }, + { + "@timestamp": "2020-11-11T21:00:00.000Z", + "event.action": "org.user.invite", + "event.dataset": "snyk.audit", + "event.module": "snyk", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 576, + "service.type": "snyk", + "snyk.audit.content.email": "someone@snyk.io", + "snyk.audit.content.isAdmin": false, + "snyk.audit.org_id": "orgid123test-5643asd234-asdfasdf", + "tags": [ + "snyk-audit", + "forwarded" + ], + "user.group.id": "groupid123test-543123-54312sadf-123ad", + "user.id": "userid123test-234sdfa2-423sdfa-2134" + }, + { + "@timestamp": "2020-11-15T06:02:45.497Z", + "event.action": "org.user.role.edit", + "event.dataset": "snyk.audit", + "event.module": "snyk", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 852, + "service.type": "snyk", + "snyk.audit.content.after": "ADMIN", + "snyk.audit.content.before": "COLLABORATOR", + "snyk.audit.content.userPublicId": "userid123test-234sdfa2-423sdfa-2134", + "snyk.audit.org_id": "orgid123test-5643asd234-asdfasdf", + "tags": [ + "snyk-audit", + "forwarded" + ], + "user.group.id": "groupid123test-543123-54312sadf-123ad", + "user.id": "userid123test-234sdfa2-423sdfa-2134" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/snyk/fields.go b/x-pack/filebeat/module/snyk/fields.go new file mode 100644 index 00000000000..db4083059e2 --- /dev/null +++ b/x-pack/filebeat/module/snyk/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package snyk + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "snyk", asset.ModuleFieldsPri, AssetSnyk); err != nil { + panic(err) + } +} + +// AssetSnyk returns asset data. +// This is the base64 encoded gzipped contents of module/snyk. +func AssetSnyk() string { + return "eJy8mMFym0gQhu9+ii5fckl0yU2HrUrZe/DW7iYVb+0eVQ00qONRD+kZkNmnT80AMkhg2Q4KF5cZmP+ju6fnH32AB2rW4KR5uALw7A2t4b79LyOXKpeerazhtysAiCOws1ll6AogZzKZW8eRDyC4o8NM4fJNSWso1FZld0fJEDpaQ0Ieu3sTKuH6K4pAbhVKVMdStOKl2m+UeqgrI6SYsGHP5Fbdi0OkIVb3mjsM9Hi5Qe9JKBuMzCCF65MqNrBnvwU0JnwQesoO04NN4t/VCUD35GoW5IGavdVXYdg8QvgtHYPA3e07t7o6ocAqY38iPUzRXJrOAM2lKwqCscUgJKdZGiJaLTacjYbmQ3SGKlx3tyFOwxh91gKF/8fwxuGmt/EZqkn8ahKsC+2F4PrEvZwnteJJ/CTMVFW/AOdzTVoz7XuoTgL8Fj3s0UG6RSkoew+J9VuwJgOUDIT2UKOpaKLuj9bpr6y9M60CzhRiWjv3cblU39zf1x/BpVbHCE+CGbvUWFcpbTzvaFI6Q388cEb3ny1BmA78lt0oFk1MqVUuWNCYpgd4qr4S0wcsCHbI4pGFdAadHktj2W926Ctl3ywXtoAfs9lJQC8Bhmoy0zhLLtEAMI6aUk5KkhLc3c7ouw0XYvVk/bUcibWGUF7ZKtoleYxSWvWwRQcJkUCnOktVok+3mJjp4nor15iJHeT8GEQgaaCK6xHbHLqqLA2HbSpwPE958dgdgtbJzeOwyOIxO+UZxy2ItpHziuLYc02QUUmSkaTNLGxVForZL8Ad5jhqtrTnEA1KUWExTffmBdo1qncuNP1CcbcLNL3WzH7evnMREOCMxHPOpIBpajVGx1tg7wZdVbAgfZZu0z10GcozBGFZuCX9xZd2SlD6XrG2+4ySs6amWGHsXEWQKkULlDSxa8y5MbZhG9jEHXUS0lgpXh+bm3/vD5O32/UMQJUYTqOJXHq7Hi+0gVDcyKdxvrHiJsZvU6lZrlz+ZHno7UDvTf9gxTZX0yy9pdg4qmnWDrwtPf3c0M/de9VR0Ka5lDDsfvGJBT38WD840UrSmK5c7S4OGk4UNfb4Kvgrls5fZwQuRRHKVnCDAsR+SwoJwd8W7iS37+GLDR6co0PrvsBQ9N1f+/+mPzee5pdtHO0CjRPPaDblZSSbktoIJdQH6dpwSuLoGqzC9Sj/19N0lfD3ivqyZHIbw276BPUm3k8Q5gsV2S+VVhGeFKfBalLH9nhbXqjLd5O3B4A2nOwAgxVLYy15O2MnxKvNqpSyzUQj+5kOF96a8BThPMLCXbHntpJ5V5bz4wUsYp+3GavYbVYzVKnS+OcV+Lkkfj0cNPrzGKmLiUT/1AhjmF7aAh3t6hk38dbfDCQ0ZdhZpW52UJSC3LP1Fqsgt7pDDzUqkxv7pBc5lCeT5VZoPKlgsMkLrucs4zCG5tirD5TPoqX7BZFu/vv9OZYfAQAA///5jfvA" +} diff --git a/x-pack/filebeat/module/snyk/module.yml b/x-pack/filebeat/module/snyk/module.yml new file mode 100644 index 00000000000..ed97d539c09 --- /dev/null +++ b/x-pack/filebeat/module/snyk/module.yml @@ -0,0 +1 @@ +--- diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/_meta/fields.yml b/x-pack/filebeat/module/snyk/vulnerabilities/_meta/fields.yml new file mode 100644 index 00000000000..4d63ad36504 --- /dev/null +++ b/x-pack/filebeat/module/snyk/vulnerabilities/_meta/fields.yml @@ -0,0 +1,118 @@ +- name: vulnerabilities + type: group + release: beta + description: > + Module for parsing Snyk project vulnerabilities. + fields: + - name: cvss3 + type: keyword + description: > + CSSv3 scores. + - name: disclosure_time + type: date + description: > + The time this vulnerability was originally disclosed to the package maintainers. + - name: exploit_maturity + type: keyword + description: > + The Snyk exploit maturity level. + - name: id + type: keyword + description: > + The vulnerability reference ID. + - name: is_ignored + type: boolean + description: > + If the vulnerability report has been ignored. + - name: is_patchable + type: boolean + description: > + If vulnerability is fixable by using a Snyk supplied patch. + - name: is_patched + type: boolean + description: > + If the vulnerability has been patched. + - name: is_pinnable + type: boolean + description: > + If the vulnerability is fixable by pinning a transitive dependency. + - name: is_upgradable + type: boolean + description: > + If the vulnerability fixable by upgrading a dependency. + - name: language + type: keyword + description: > + The package's programming language. + - name: package + type: keyword + description: > + The package identifier according to its package manager. + - name: package_manager + type: keyword + description: > + The package manager. + - name: patches + type: flattened + description: > + Patches required to resolve the issue created by Snyk. + - name: priority_score + type: long + description: > + The CVS priority score. + - name: publication_time + type: date + description: > + The vulnerability publication time. + - name: jira_issue_url + type: keyword + description: > + Link to the related Jira issue. + - name: original_severity + type: long + description: > + The original severity of the vulnerability. + - name: reachability + type: keyword + description: > + If the vulnerable function from the library is used in the code scanned. Can either be No Info, Potentially reachable and Reachable. + - name: title + type: keyword + description: > + The issue title. + - name: type + type: keyword + description: > + The issue type. Can be either "license" or "vulnerability". + - name: unique_severities_list + type: keyword + description: > + A list of related unique severities. + - name: version + type: keyword + description: > + The package version this issue is applicable to. + - name: introduced_date + type: date + description: > + The date the vulnerability was initially found. + - name: is_fixed + type: boolean + description: > + If the related vulnerability has been resolved. + - name: credit + type: keyword + description: > + Reference to the person that original found the vulnerability. + - name: semver + type: flattened + description: > + One or more semver ranges this issue is applicable to. The format varies according to package manager. + - name: identifiers.alternative + type: keyword + description: > + Additional vulnerability identifiers. + - name: identifiers.cwe + type: keyword + description: > + CWE vulnerability identifiers. diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml new file mode 100644 index 00000000000..7ce5c570372 --- /dev/null +++ b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml @@ -0,0 +1,99 @@ +{{ if eq .input "httpjson" }} + +type: httpjson +config_version: "2" +interval: {{ .interval }} + +request.url: {{ .url }} +request.method: POST +request.ssl: {{ .ssl | tojson }} +request.transforms: +- set: + target: header.Authorization + value: token {{ .api_token }} +- set: + target: url.params.to + value: '[[ formatDate (now) "2006-01-02" ]]' +- set: + target: url.params.from + value: '[[ formatDate (now (parseDuration .cursor.interval)) "2006-01-02" ]]' + default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "2006-01-02" ]]' +cursor: + interval: + value: "-24h" +request.timeout: 120s +request.body: + filters: + orgs: + {{ range .orgs }} + - {{ . }} + {{ end }} + severity: + {{ range .included_severity }} + - {{ . }} + {{ end }} + exploitMaturity: + {{ range .exploit_maturity }} + - {{ . }} + {{ end }} + types: + {{ range .types }} + - {{ . }} + {{ end }} + languages: + {{ range .languages }} + - {{ . }} + {{ end }} + identifier: {{ .identifier | tojson}} + ignored: {{ .ignored }} + patched: {{ .patched }} + fixable: {{ .fixable }} + isFixed: {{ .is_fixed }} + isUpgradable: {{ .is_upgradeable }} + isPatchable: {{ .is_patchable }} + isPinnable: {{ .is_pinnable }} + priorityScore: + min: {{ .min_priority_score }} + max: {{ .max_priority_score }} + +response.pagination: +- set: + target: url.value + value: '[[ getRFC5988Link "next" .last_response.header.Link ]]' + +response.split: + target: body.results + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_locale: ~ + - decode_json_fields: + fields: [message] + target: json + - fingerprint: + fields: ["json.issue.id"] + target_field: "@metadata._id" + - script: + lang: javascript + id: my_filter + source: > + function process(event) { + event.Put("@metadata.op_type", "index"); + } + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/ingest/pipeline.yml b/x-pack/filebeat/module/snyk/vulnerabilities/ingest/pipeline.yml new file mode 100644 index 00000000000..bbb46e5cc4d --- /dev/null +++ b/x-pack/filebeat/module/snyk/vulnerabilities/ingest/pipeline.yml @@ -0,0 +1,181 @@ +description: Pipeline for Snyk vulnerability logs +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- set: + field: vulnerability.classification + value: CVSS +- set: + field: vulnerability.category + value: Github +- set: + field: vulnerability.scanner.vendor + value: Snyk +- script: + lang: painless + if: ctx?.json != null + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); +- rename: + field: json + target_field: snyk + ignore_missing: true +- rename: + field: snyk.issue + target_field: snyk.vulnerabilities + ignore_missing: true +- set: + field: vulnerability.score.version + value: "3.0" +- set: + field: vulnerability.enumeration + value: CVE + if: ctx?.snyk?.vulnerabilities?.identifiers?.CVE != null +- set: + field: vulnerability.enumeration + value: SNYK + if: ctx?.snyk?.vulnerabilities?.identifiers?.CVE == null && ctx?.snyk?.vulnerabilities?.identifiers?.ALTERNATIVE != null +- rename: + field: snyk.vulnerabilities.description + target_field: vulnerability.description + ignore_missing: true +- rename: + field: snyk.vulnerabilities.identifiers.CVE + target_field: vulnerability.id + ignore_missing: true + if: ctx?.snyk?.vulnerabilities?.identifiers?.CVE != null +- rename: + field: snyk.vulnerabilities.identifiers.ALTERNATIVE + target_field: vulnerability.id + ignore_missing: true + if: ctx?.vulnerability?.id == null && ctx?.snyk?.vulnerabilities?.identifiers?.ALTERNATIVE != null +- rename: + field: snyk.vulnerabilities.cvssScore + target_field: vulnerability.score.base + ignore_missing: true +- rename: + field: snyk.vulnerabilities.severity + target_field: vulnerability.severity + ignore_missing: true +- rename: + field: snyk.vulnerabilities.CVSSv3 + target_field: snyk.vulnerabilities.cvss3 + ignore_missing: true +- rename: + field: snyk.vulnerabilities.disclosureTime + target_field: snyk.vulnerabilities.disclosure_time + ignore_missing: true +- rename: + field: snyk.vulnerabilities.exploitMaturity + target_field: snyk.vulnerabilities.exploit_maturity + ignore_missing: true +- rename: + field: snyk.vulnerabilities.identifiers.ALTERNATIVE + target_field: snyk.vulnerabilities.identifiers.alternative + ignore_missing: true +- rename: + field: snyk.vulnerabilities.identifiers.CWE + target_field: snyk.vulnerabilities.identifiers.cwe + ignore_missing: true +- rename: + field: snyk.vulnerabilities.isIgnored + target_field: snyk.vulnerabilities.is_ignored + ignore_missing: true +- rename: + field: snyk.vulnerabilities.isPatchable + target_field: snyk.vulnerabilities.is_patchable + ignore_missing: true +- rename: + field: snyk.vulnerabilities.isPatched + target_field: snyk.vulnerabilities.is_patched + ignore_missing: true +- rename: + field: snyk.vulnerabilities.isPinnable + target_field: snyk.vulnerabilities.is_pinnable + ignore_missing: true +- rename: + field: snyk.vulnerabilities.isUpgradable + target_field: snyk.vulnerabilities.is_upgradable + ignore_missing: true +- rename: + field: snyk.vulnerabilities.priorityScore + target_field: snyk.vulnerabilities.priority_score + ignore_missing: true +- rename: + field: snyk.vulnerabilities.publicationTime + target_field: snyk.vulnerabilities.publication_time + ignore_missing: true +- rename: + field: snyk.vulnerabilities.uniqueSeveritiesList + target_field: snyk.vulnerabilities.unique_severities_list + ignore_missing: true +- rename: + field: snyk.vulnerabilities.packageManager + target_field: snyk.vulnerabilities.package_manager + ignore_missing: true +- rename: + field: snyk.vulnerabilities.jiraIssueUrl + target_field: snyk.vulnerabilities.jira_issue_url + ignore_missing: true +- rename: + field: snyk.vulnerabilities.originalSeverity + target_field: snyk.vulnerabilities.original_severity + ignore_missing: true +- rename: + field: snyk.isFixed + target_field: snyk.vulnerabilities.is_fixed + ignore_missing: true +- rename: + field: snyk.introducedDate + target_field: snyk.vulnerabilities.introduced_date + ignore_missing: true +- rename: + field: snyk.vulnerabilities.url + target_field: vulnerability.reference + ignore_missing: true +- foreach: + field: snyk.vulnerabilities.patches + processor: + rename: + field: "{{ _ingest._value.modificationTime }}" + target_field: "{{ _ingest._value.modification_time }}" + ignore_missing: true + ignore_failure: true + if: ctx?.snyk?.vulnerabilities?.patches != null +- foreach: + field: snyk.projects + processor: + append: + field: snyk.related.projects + value: "{{_ingest._value.name }}" + ignore_failure: true + ignore_failure: true + if: ctx?.snyk?.projects != null +- remove: + field: + - message + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml b/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml new file mode 100644 index 00000000000..391333380dc --- /dev/null +++ b/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml @@ -0,0 +1,67 @@ +module_version: 1.0 + +var: + - name: tags + default: [snyk-vulnerabilities, forwarded] + - name: input + default: httpjson + - name: first_interval + default: "24h" + - name: interval + default: 24h + - name: url + default: https://snyk.io/api/v1/reporting/issues/?page=1&perPage=10&sortBy=issueTitle&order=asc&groupBy=issue + - name: api_token + - name: ssl + - name: orgs + - name: included_severity + default: + - high + - medium + - low + - name: exploit_maturity + default: + - mature + - proof-of-concept + - no-known-exploit + - no-data + - name: types + default: + - vuln + - license + - name: languages + default: + - javascript + - ruby + - java + - scala + - python + - golang + - php + - dotnet + - swift + - docker + - name: identifier + default: "" + - name: ignored + default: false + - name: patched + default: false + - name: fixable + default: false + - name: is_fixed + default: false + - name: is_upgradeable + default: false + - name: is_patchable + default: false + - name: is_pinnable + default: false + - name: min_priority_score + default: 0 + - name: max_priority_score + default: 1000 + +ingest_pipeline: + - ingest/pipeline.yml +input: config/config.yml diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/test/snyk_vulns.ndjson.log b/x-pack/filebeat/module/snyk/vulnerabilities/test/snyk_vulns.ndjson.log new file mode 100644 index 00000000000..f5a86646d8f --- /dev/null +++ b/x-pack/filebeat/module/snyk/vulnerabilities/test/snyk_vulns.ndjson.log @@ -0,0 +1,4 @@ +{"issue":{"url":"https://snyk.io/vuln/npm:ejs:20161128","id":"npm:ejs:20161128","title":"Arbitrary Code Execution","type":"vuln","package":"ejs","version":"0.8.8","severity":"high","originalSeverity":null,"uniqueSeveritiesList":["high"],"language":"js","packageManager":"npm","semver":{"vulnerable":["<2.5.3"]},"isIgnored":false,"publicationTime":"2016-11-28T18:44:12.000Z","disclosureTime":"2016-11-27T22:00:00.000Z","isUpgradable":false,"isPatchable":false,"isPinnable":false,"identifiers":{"CVE":[],"CWE":["CWE-94"],"ALTERNATIVE":["SNYK-JS-EJS-10218"]},"credit":["Snyk Security Research Team"],"CVSSv3":"CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H","cvssScore":"8.1","patches":[{"id":"patch:npm:ejs:20161128:0","urls":["https://snyk-patches.s3.amazonaws.com/npm/ejs/20161128/ejs_20161128_0_0_3d447c5a335844b25faec04b1132dbc721f9c8f6.patch"],"version":"<2.5.3 >=2.2.4","comments":[],"modificationTime":"2019-12-03T11:40:45.851976Z"}],"isPatched":false,"exploitMaturity":"no-known-exploit","reachability":"No Info","priorityScore":4.05,"jiraIssueUrl":null},"isFixed":false,"introducedDate":"2020-04-07","projects":[{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"username/reponame","source":"github","packageManager":"npm","targetFile":"package.json"},{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"someotheruser/someotherreponame","source":"github","packageManager":"npm","targetFile":"folder1/package.json"},{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"projectname","source":"cli","packageManager":"npm","targetFile":"package.json"}]} +{"issue":{"url":"https://snyk.io/vuln/npm:ejs:20161128","id":"npm:ejs:20161121","title":"Arbitrary Code Execution","type":"vuln","package":"ejs","version":"0.8.8","severity":"high","originalSeverity":null,"uniqueSeveritiesList":["high"],"language":"js","packageManager":"npm","semver":{"vulnerable":["<2.5.3"]},"isIgnored":false,"publicationTime":"2016-11-28T18:44:12.000Z","disclosureTime":"2016-11-27T22:00:00.000Z","isUpgradable":false,"isPatchable":false,"isPinnable":false,"identifiers":{"CVE":["CVE-2017-1000228"],"CWE":["CWE-94"],"ALTERNATIVE":["SNYK-JS-EJS-10218"]},"credit":["Snyk Security Research Team"],"CVSSv3":"CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H","cvssScore":"8.1","patches":[{"id":"patch:npm:ejs:20161128:0","urls":["https://snyk-patches.s3.amazonaws.com/npm/ejs/20161128/ejs_20161128_0_0_3d447c5a335844b25faec04b1132dbc721f9c8f6.patch"],"version":"<2.5.3 >=2.2.4","comments":[],"modificationTime":"2019-12-03T11:40:45.851976Z"}],"isPatched":false,"exploitMaturity":"no-known-exploit","reachability":"No Info","priorityScore":619,"jiraIssueUrl":null},"isFixed":false,"introducedDate":"2020-11-13","projects":[{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"username/reponame","source":"github","packageManager":"npm","targetFile":"package.json"},{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"someotheruser/someotherreponame","source":"github","packageManager":"npm","targetFile":"folder1/package.json"},{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"projectname","source":"cli","packageManager":"npm","targetFile":"package.json"}]} +{"issue":{"url":"https://snyk.io/vuln/SNYK-GOLANG-GITHUBCOMSATORIGOUUID-72488","id":"SNYK-GOLANG-GITHUBCOMSATORIGOUUID-72488","title":"Insecure Randomness","type":"vuln","package":"github.com/satori/go.uuid","version":"#000000000000","severity":"high","originalSeverity":null,"uniqueSeveritiesList":["high"],"language":"golang","packageManager":"golang","semver":{"vulnerable":["=1.2.0"],"hashesRange":[">=0ef6afb2f6cdd6cdaeee3885a95099c63f18fc8c =2.2.4" + } + ], + "snyk.vulnerabilities.priority_score": 4.05, + "snyk.vulnerabilities.publication_time": "2016-11-28T18:44:12.000Z", + "snyk.vulnerabilities.reachability": "No Info", + "snyk.vulnerabilities.semver.vulnerable": [ + "<2.5.3" + ], + "snyk.vulnerabilities.title": "Arbitrary Code Execution", + "snyk.vulnerabilities.type": "vuln", + "snyk.vulnerabilities.unique_severities_list": [ + "high" + ], + "snyk.vulnerabilities.version": "0.8.8", + "tags": [ + "snyk-vulnerabilities", + "forwarded" + ], + "vulnerability.category": "Github", + "vulnerability.classification": "CVSS", + "vulnerability.enumeration": "CVE", + "vulnerability.id": [], + "vulnerability.reference": "https://snyk.io/vuln/npm:ejs:20161128", + "vulnerability.scanner.vendor": "Snyk", + "vulnerability.score.base": "8.1", + "vulnerability.score.version": "3.0", + "vulnerability.severity": "high" + }, + { + "event.dataset": "snyk.vulnerabilities", + "event.module": "snyk", + "event.timezone": "-02:00", + "fileset.name": "vulnerabilities", + "input.type": "log", + "log.offset": 1639, + "service.type": "snyk", + "snyk.projects": [ + { + "id": "projectid", + "name": "username/reponame", + "packageManager": "npm", + "source": "github", + "targetFile": "package.json", + "url": "https://snyk.io/org/orgname/project/projectid" + }, + { + "id": "projectid", + "name": "someotheruser/someotherreponame", + "packageManager": "npm", + "source": "github", + "targetFile": "folder1/package.json", + "url": "https://snyk.io/org/orgname/project/projectid" + }, + { + "id": "projectid", + "name": "projectname", + "packageManager": "npm", + "source": "cli", + "targetFile": "package.json", + "url": "https://snyk.io/org/orgname/project/projectid" + } + ], + "snyk.related.projects": [ + "username/reponame", + "someotheruser/someotherreponame", + "projectname" + ], + "snyk.vulnerabilities.credit": [ + "Snyk Security Research Team" + ], + "snyk.vulnerabilities.cvss3": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + "snyk.vulnerabilities.disclosure_time": "2016-11-27T22:00:00.000Z", + "snyk.vulnerabilities.exploit_maturity": "no-known-exploit", + "snyk.vulnerabilities.id": "npm:ejs:20161121", + "snyk.vulnerabilities.identifiers.alternative": [ + "SNYK-JS-EJS-10218" + ], + "snyk.vulnerabilities.identifiers.cwe": [ + "CWE-94" + ], + "snyk.vulnerabilities.introduced_date": "2020-11-13", + "snyk.vulnerabilities.is_fixed": false, + "snyk.vulnerabilities.is_ignored": false, + "snyk.vulnerabilities.is_patchable": false, + "snyk.vulnerabilities.is_patched": false, + "snyk.vulnerabilities.is_pinnable": false, + "snyk.vulnerabilities.is_upgradable": false, + "snyk.vulnerabilities.language": "js", + "snyk.vulnerabilities.package": "ejs", + "snyk.vulnerabilities.package_manager": "npm", + "snyk.vulnerabilities.patches": [ + { + "comments": [], + "id": "patch:npm:ejs:20161128:0", + "modificationTime": "2019-12-03T11:40:45.851976Z", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/ejs/20161128/ejs_20161128_0_0_3d447c5a335844b25faec04b1132dbc721f9c8f6.patch" + ], + "version": "<2.5.3 >=2.2.4" + } + ], + "snyk.vulnerabilities.priority_score": 619, + "snyk.vulnerabilities.publication_time": "2016-11-28T18:44:12.000Z", + "snyk.vulnerabilities.reachability": "No Info", + "snyk.vulnerabilities.semver.vulnerable": [ + "<2.5.3" + ], + "snyk.vulnerabilities.title": "Arbitrary Code Execution", + "snyk.vulnerabilities.type": "vuln", + "snyk.vulnerabilities.unique_severities_list": [ + "high" + ], + "snyk.vulnerabilities.version": "0.8.8", + "tags": [ + "snyk-vulnerabilities", + "forwarded" + ], + "vulnerability.category": "Github", + "vulnerability.classification": "CVSS", + "vulnerability.enumeration": "CVE", + "vulnerability.id": [ + "CVE-2017-1000228" + ], + "vulnerability.reference": "https://snyk.io/vuln/npm:ejs:20161128", + "vulnerability.scanner.vendor": "Snyk", + "vulnerability.score.base": "8.1", + "vulnerability.score.version": "3.0", + "vulnerability.severity": "high" + }, + { + "event.dataset": "snyk.vulnerabilities", + "event.module": "snyk", + "event.timezone": "-02:00", + "fileset.name": "vulnerabilities", + "input.type": "log", + "log.offset": 3295, + "service.type": "snyk", + "snyk.projects": [ + { + "id": "projectid", + "name": "username/reponame", + "packageManager": "npm", + "source": "github", + "targetFile": "package.json", + "url": "https://snyk.io/org/orgname/project/projectid" + }, + { + "id": "projectid", + "name": "someotheruser/someotherreponame", + "packageManager": "npm", + "source": "github", + "targetFile": "folder1/package.json", + "url": "https://snyk.io/org/orgname/project/projectid" + }, + { + "id": "projectid", + "name": "projectname", + "packageManager": "npm", + "source": "cli", + "targetFile": "package.json", + "url": "https://snyk.io/org/orgname/project/projectid" + } + ], + "snyk.related.projects": [ + "username/reponame", + "someotheruser/someotherreponame", + "projectname" + ], + "snyk.vulnerabilities.credit": [ + "josselin-c" + ], + "snyk.vulnerabilities.cvss3": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H", + "snyk.vulnerabilities.disclosure_time": "2018-03-23T08:57:24.000Z", + "snyk.vulnerabilities.exploit_maturity": "no-known-exploit", + "snyk.vulnerabilities.id": "SNYK-GOLANG-GITHUBCOMSATORIGOUUID-72488", + "snyk.vulnerabilities.identifiers.cwe": [ + "CWE-338" + ], + "snyk.vulnerabilities.introduced_date": "2020-11-17", + "snyk.vulnerabilities.is_fixed": false, + "snyk.vulnerabilities.is_ignored": false, + "snyk.vulnerabilities.is_patchable": false, + "snyk.vulnerabilities.is_patched": false, + "snyk.vulnerabilities.is_pinnable": false, + "snyk.vulnerabilities.is_upgradable": false, + "snyk.vulnerabilities.language": "golang", + "snyk.vulnerabilities.package": "github.com/satori/go.uuid", + "snyk.vulnerabilities.package_manager": "golang", + "snyk.vulnerabilities.patches": [], + "snyk.vulnerabilities.priority_score": 405, + "snyk.vulnerabilities.publication_time": "2018-10-24T08:56:41.000Z", + "snyk.vulnerabilities.reachability": "No Info", + "snyk.vulnerabilities.semver.hashesRange": [ + ">=0ef6afb2f6cdd6cdaeee3885a95099c63f18fc8c