diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 49a3a79eece..ee796c027f1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -56,6 +56,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Adds `date_cursor` option to httpjson input. {pull}19483[19483] - Adds Gsuite module with SAML support. {pull}19329[19329] - Adds Gsuite User Accounts support. {pull}19329[19329] +- Adds Gsuite Login audit support. {pull}19702[19702] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index dbd4d7d7214..0a380cc8834 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -61464,6 +61464,58 @@ type: keyword -- +*`gsuite.login.affected_email_address`*:: ++ +-- +type: keyword + +-- + +*`gsuite.login.challenge_method`*:: ++ +-- +Login challenge method. For a list of possible values refer to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login. + + +type: keyword + +-- + +*`gsuite.login.failure_type`*:: ++ +-- +Login failure type. For a list of possible values refer to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login. + + +type: keyword + +-- + +*`gsuite.login.type`*:: ++ +-- +Login credentials type. For a list of possible values refer to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login. + + +type: keyword + +-- + +*`gsuite.login.is_second_factor`*:: ++ +-- +type: boolean + +-- + +*`gsuite.login.is_suspicious`*:: ++ +-- +type: boolean + +-- + + *`gsuite.saml.application_name`*:: + -- diff --git a/filebeat/docs/modules/gsuite.asciidoc b/filebeat/docs/modules/gsuite.asciidoc index 0870f008c79..565617a40d6 100644 --- a/filebeat/docs/modules/gsuite.asciidoc +++ b/filebeat/docs/modules/gsuite.asciidoc @@ -23,6 +23,7 @@ It is compatible with a subset of applications under the https://developers.goog - https://developers.google.com/admin-sdk/reports/v1/appendix/activity/saml[SAML Audit Activity Events] - https://developers.google.com/admin-sdk/reports/v1/appendix/activity/user-accounts[User Accounts Activity Events] +- https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login[Login Audit Activity Events] === Configure the module @@ -49,6 +50,10 @@ you can set up your module: enabled: true var.jwt_file: "./credentials_file.json" var.delegated_account: "user@example.com" + login: + enabled: true + var.jwt_file: "./credentials_file.json" + var.delegated_account: "user@example.com" ---- Every fileset has the following configuration options: diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 7174142dc1b..e7bf1740d24 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -711,11 +711,30 @@ filebeat.modules: #-------------------------------- Gsuite Module -------------------------------- - module: gsuite - # All logs saml: enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s user_accounts: enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s + login: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s #------------------------------- HAProxy Module ------------------------------- - module: haproxy diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index 63d20221de4..2be0eb6f211 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -46,6 +46,7 @@ type Pagination struct { Header *Header `config:"header"` IDField string `config:"id_field"` RequestField string `config:"req_field"` + URLField string `config:"url_field"` URL string `config:"url"` } @@ -69,7 +70,7 @@ type RateLimit struct { type DateCursor struct { Enabled *bool `config:"enabled"` - Field string `config:"field" validate:"required"` + Field string `config:"field"` URLField string `config:"url_field" validate:"required"` ValueTemplate *Template `config:"value_template"` DateFormat string `config:"date_format"` @@ -131,9 +132,6 @@ func (c *config) Validate() error { } } if c.Pagination != nil { - if c.DateCursor.IsEnabled() { - return errors.Errorf("invalid configuration: date_cursor cannnot be set in combination with other pagination mechanisms") - } if c.Pagination.Header != nil { if c.Pagination.RequestField != "" || c.Pagination.IDField != "" || len(c.Pagination.ExtraBodyContent) > 0 { return errors.Errorf("invalid configuration: both pagination.header and pagination.req_field or pagination.id_field or pagination.extra_body_content cannot be set simultaneously") diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 904702ee116..c3486aedda4 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -351,17 +351,6 @@ func TestConfigOauth2Validation(t *testing.T) { "url": "localhost", }, }, - { - name: "date_cursor must fail in combination with pagination", - expectedErr: "invalid configuration: date_cursor cannnot be set in combination with other pagination mechanisms accessing config", - input: map[string]interface{}{ - "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo"}, - "pagination": map[string]interface{}{ - "header": map[string]interface{}{"field_name": "foo", "regex_pattern": "bar"}, - }, - "url": "localhost", - }, - }, { name: "date_cursor.date_format will fail if invalid", expectedErr: "invalid configuration: date_format is not a valid date layout accessing 'date_cursor'", diff --git a/x-pack/filebeat/input/httpjson/httpjson_test.go b/x-pack/filebeat/input/httpjson/httpjson_test.go index 75374404eea..a6ebd16ad5d 100644 --- a/x-pack/filebeat/input/httpjson/httpjson_test.go +++ b/x-pack/filebeat/input/httpjson/httpjson_test.go @@ -336,11 +336,22 @@ func TestCreateRequestInfoFromBody(t *testing.T) { "id": 100, } extraBodyContent := common.MapStr{"extra_body": "abc"} - ri, err := createRequestInfoFromBody(common.MapStr(m), "id", "pagination_id", extraBodyContent, "https://test-123", &RequestInfo{ - URL: "", - ContentMap: common.MapStr{}, - Headers: common.MapStr{}, - }) + config := &Pagination{ + IDField: "id", + RequestField: "pagination_id", + ExtraBodyContent: extraBodyContent, + URL: "https://test-123", + } + ri, err := createRequestInfoFromBody( + config, + common.MapStr(m), + common.MapStr(m), + &RequestInfo{ + URL: "", + ContentMap: common.MapStr{}, + Headers: common.MapStr{}, + }, + ) if ri.URL != "https://test-123" { t.Fatal("Failed to test createRequestInfoFromBody. URL should be https://test-123.") } diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 3ed396db6c8..50677876b1f 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "io/ioutil" "net" @@ -319,19 +320,33 @@ func (in *HttpjsonInput) applyRateLimit(ctx context.Context, header http.Header, } // createRequestInfoFromBody creates a new RequestInfo for a new HTTP request in pagination based on HTTP response body -func createRequestInfoFromBody(m common.MapStr, idField string, requestField string, extraBodyContent common.MapStr, url string, ri *RequestInfo) (*RequestInfo, error) { - v, err := m.GetValue(idField) +func createRequestInfoFromBody(config *Pagination, response, last common.MapStr, ri *RequestInfo) (*RequestInfo, error) { + // we try to get it from last element, if not found, from the original response + v, err := last.GetValue(config.IDField) + if err == common.ErrKeyNotFound { + v, err = response.GetValue(config.IDField) + } + + if err == common.ErrKeyNotFound { + return nil, nil + } + if err != nil { - if err == common.ErrKeyNotFound { - return nil, nil - } else { - return nil, errors.Wrapf(err, "failed to retrieve id_field for pagination") - } + return nil, errors.Wrapf(err, "failed to retrieve id_field for pagination") } - if requestField != "" { - ri.ContentMap.Put(requestField, v) - if url != "" { - ri.URL = url + + if config.RequestField != "" { + ri.ContentMap.Put(config.RequestField, v) + if config.URL != "" { + ri.URL = config.URL + } + } else if config.URLField != "" { + url, err := url.Parse(ri.URL) + if err == nil { + q := url.Query() + q.Set(config.URLField, fmt.Sprint(v)) + url.RawQuery = q.Encode() + ri.URL = url.String() } } else { switch vt := v.(type) { @@ -341,8 +356,8 @@ func createRequestInfoFromBody(m common.MapStr, idField string, requestField str return nil, errors.New("pagination ID is not of string type") } } - if len(extraBodyContent) > 0 { - ri.ContentMap.Update(extraBodyContent) + if len(config.ExtraBodyContent) > 0 { + ri.ContentMap.Update(common.MapStr(config.ExtraBodyContent)) } return ri, nil } @@ -350,6 +365,12 @@ func createRequestInfoFromBody(m common.MapStr, idField string, requestField str // processHTTPRequest processes HTTP request, and handles pagination if enabled func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Client, ri *RequestInfo) error { ri.URL = in.getURL() + + var ( + m, v interface{} + response, mm map[string]interface{} + ) + for { req, err := in.createHTTPRequest(ctx, ri) if err != nil { @@ -375,8 +396,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl } return errors.Errorf("http request was unsuccessful with a status code %d", msg.StatusCode) } - var m, v interface{} - var mm map[string]interface{} + err = json.Unmarshal(responseData, &m) if err != nil { in.log.Debug("failed to unmarshal http.response.body", string(responseData)) @@ -390,6 +410,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl return err } case map[string]interface{}: + response = obj if in.config.JSONObjects == "" { mm, err = in.processEventArray([]interface{}{obj}) if err != nil { @@ -399,7 +420,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl v, err = common.MapStr(obj).GetValue(in.config.JSONObjects) if err != nil { if err == common.ErrKeyNotFound { - return nil + break } return err } @@ -417,6 +438,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl in.log.Debug("http.response.body is not a valid JSON object", string(responseData)) return errors.Errorf("http.response.body is not a valid JSON object, but a %T", obj) } + if mm != nil && in.config.Pagination.IsEnabled() { if in.config.Pagination.Header != nil { // Pagination control using HTTP Header @@ -426,7 +448,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl } if ri.URL == url || url == "" { in.log.Info("Pagination finished.") - return nil + break } ri.URL = url if err = in.applyRateLimit(ctx, header, in.config.RateLimit); err != nil { @@ -436,12 +458,12 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl continue } else { // Pagination control using HTTP Body fields - ri, err = createRequestInfoFromBody(common.MapStr(mm), in.config.Pagination.IDField, in.config.Pagination.RequestField, common.MapStr(in.config.Pagination.ExtraBodyContent), in.config.Pagination.URL, ri) + ri, err = createRequestInfoFromBody(in.config.Pagination, common.MapStr(response), common.MapStr(mm), ri) if err != nil { return err } if ri == nil { - return nil + break } if err = in.applyRateLimit(ctx, header, in.config.RateLimit); err != nil { return err @@ -450,11 +472,14 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl continue } } - if mm != nil && in.config.DateCursor.IsEnabled() { - in.advanceCursor(common.MapStr(mm)) - } - return nil + break + } + + if mm != nil && in.config.DateCursor.IsEnabled() { + in.advanceCursor(common.MapStr(mm)) } + + return nil } func (in *HttpjsonInput) getURL() string { @@ -496,6 +521,11 @@ func (in *HttpjsonInput) getURL() string { } func (in *HttpjsonInput) advanceCursor(m common.MapStr) { + if in.config.DateCursor.Field == "" { + in.nextCursorValue = time.Now().UTC().Format(in.config.DateCursor.GetDateFormat()) + return + } + v, err := m.GetValue(in.config.DateCursor.Field) if err != nil { in.log.Warnf("date_cursor field: %q", err) @@ -505,6 +535,7 @@ func (in *HttpjsonInput) advanceCursor(m common.MapStr) { case string: _, err := time.Parse(in.config.DateCursor.GetDateFormat(), t) if err != nil { + in.log.Warn("date_cursor field does not have the expected layout") return } in.nextCursorValue = t diff --git a/x-pack/filebeat/module/gsuite/_meta/config.yml b/x-pack/filebeat/module/gsuite/_meta/config.yml index 5301bb8567a..65b289a5010 100644 --- a/x-pack/filebeat/module/gsuite/_meta/config.yml +++ b/x-pack/filebeat/module/gsuite/_meta/config.yml @@ -1,6 +1,25 @@ - module: gsuite - # All logs saml: enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s user_accounts: enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s + login: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s diff --git a/x-pack/filebeat/module/gsuite/_meta/docs.asciidoc b/x-pack/filebeat/module/gsuite/_meta/docs.asciidoc index 4af4410b17e..9af088e36f9 100644 --- a/x-pack/filebeat/module/gsuite/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/gsuite/_meta/docs.asciidoc @@ -18,6 +18,7 @@ It is compatible with a subset of applications under the https://developers.goog - https://developers.google.com/admin-sdk/reports/v1/appendix/activity/saml[SAML Audit Activity Events] - https://developers.google.com/admin-sdk/reports/v1/appendix/activity/user-accounts[User Accounts Activity Events] +- https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login[Login Audit Activity Events] === Configure the module @@ -44,6 +45,10 @@ you can set up your module: enabled: true var.jwt_file: "./credentials_file.json" var.delegated_account: "user@example.com" + login: + enabled: true + var.jwt_file: "./credentials_file.json" + var.delegated_account: "user@example.com" ---- Every fileset has the following configuration options: diff --git a/x-pack/filebeat/module/gsuite/fields.go b/x-pack/filebeat/module/gsuite/fields.go index 5003648f341..b0cebd8b1fd 100644 --- a/x-pack/filebeat/module/gsuite/fields.go +++ b/x-pack/filebeat/module/gsuite/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGsuite returns asset data. // This is the base64 encoded gzipped contents of module/gsuite. func AssetGsuite() string { - return "eJzMVU2PIzUQvedXlJYD0orpCI45IEUorNDOMKvJLmKFUKfSLqdLcbsau5yd5tcju5Mhk2TExwxofWq5vl49v6q+gi0NM9jExEoTAGV1NINX48WrCYCh2ATulcXP4NsJAOy94UZMcjnIMjkTZ8V2BR47OsqYjyGLyWldHGdg0cWDSYc+ewdJ/YPzWcF83oxFY08NW272RavJg8ONBAL2VkKHORhwLUlPA6BBD2sCK8kbQIVWtY+z6dTQjpz0FGK1Edk4qhrppmg69lfRbKeBegkap7uvp4EsBfINTbFR3rEyxanjqHssx3wcc4KNSqhyxw+mAwFbGj5JMEf3T9CQz/uWShiI3eecPLL/hC7RodPZIxPA6w/Lxd3rGcy9aEsBUqQA7EFbgogdgZEO2VenYYuf3y/ufpxf14f4MVKSRjZUwp+IfLv4WPy9+Ks2degPoC/zs6XhefTcejdAHyiSV/jUkofVn8yvgCOs3i4+rir4bpRChr5qxMfUUai3NKwysfk20G+JokoAKwFu50lb+Ob6FubvfjjYIkgA9MCGvLJlGn2DrEUBm0aS13jeKu3I6wtL4c2yTEhJ/RV02PdkwAbpYMVKXfzl16rY8seeivHZJfCGPTrocXCC5vELLrBpwbKjSFo01eKOAMGwLWOgkA1iYTfKLrfPGUCeR0OK7P6XqcuH7rHr8wLDZFi/2DsOZ/Rv2ZuXI36UQ5QUGjohPhf6mzzffMZ8Sdig59/LXq3GMX8efSOBYybQFjWPJVpLjZKB9bAfv9zMl3E/LWeoInbuDMbxrwQuLOPjBNj3jpvSVZ1vTiA+1ddf9gawxM7B8t1xhVKzuojDIrsUqD7ZB8/EcC0b9ofcJVEF3+dtBVkFWbi9xMhrR4fRLZIBlX8jt6x6b/j+oLZhmp/ncr/sWRmVTL0eXq7fu3Ej55+SheX85howaZvX8vgAl7FI2CTPWveo7cthAfhQ/o5j8suVo6KmWDdiLj+6E7/5h6rLTY9pIad9oi414k3t8tvW/xWIUgNKjceI/ggAAP//yAgQFA==" + return "eJzkVlFvGzcMfs+vILqHAcViY3v0w4Bg8IqhyVrE7bBiGM60xPMR1ok3iXJ7+/WDdL7EcWysW9whwPRkiORHfh9J+S5hQ/0M1jGx0gWAsjqawYvh4sUFgKVoAnfK4mfw/QUA7LzhRmxyOahmcjbOiu0SPLa0h5iPpRqT06o4zqBGF0eT9l32DpK6O+dHCfN5NSSNHRmu2eySTi7uHG4kELCvJbSYgwFXkvQwAAx6WBHUkrwFVGhUuzibTi1tyUlHIU7WImtHEyPtFG3L/jLazTRQJ0HjdPvtNFBNgbyhKRrlLStTnDqOuqtlX499TdCohElmfGcaBdhQ/1GC3bs/IUM+7xoqYSD1DvPigf0XdIlGprMHJoCX7xfz25czuPKiDQVIkQKwB20IIrYEVlpkPzkMm//6bn7789V1NcYPkZI0sqUSfiLy9fxD8ffiL5vUoh+LPq7PhvqnyfPGux66QJG8wseGPCzvlV8CR1i+nn9YTuCHYRRy6UsjPqaWQrWhfpmFzbeB/kgUVQLUEuDNVdIGvrt+A1dvfxptESQAemBLXrlmGnyDrEQBjZHkNT6mSlvyeuZReLUoG1Kgv4EWu44s1EFaWLJSG3/7fVJs+cdOiqHtEnjNHh102DtB+7CDczQN1OwokpaZanBLgGC5LmugkA1Sw3YYu0yfcwF5Hy0psvtPti4f+oRtlx8wTJb1q51j/0j+DXt7PuGHcYiSgqED4XOiz9T55hnrJWGNnv8s7+pkWPOnyTcIOCCBNqh5LbGuyShZWPW79ctkvo67bXlUlZP1kTr2/0vgyGu8jzBmrKhFdhVaGyjGg0pP0buHMQ06R35NVUvaiP1sgL/RCOA6U7zHhwF/Aj/mVwdyN/MAdhIjrxyNK1haDyr/Zmzy9HrLn8ap6adF5slR4jWyS4Gqg4fsLKR32AXo2RD+EkRNoPLngS4+L7Icq0hGvK3q8gd6lPhKxBH6kwApdmxY0vGtOoweIyO27mmb3XWOTXmwqnxzvqYtsHWweLufoeT8P21Ibs+JmfGsjPlJXfXn43s7fGzl780aFlc314BJm7w0QwOO1yJhnTxr1aE256sF4H358B3Aj2eOippiZcQeb7oTv/6HU5dJD7CQYU/kHdbV5d5WX6qIkgNKjocV/RUAAP//BvI0zA==" } diff --git a/x-pack/filebeat/module/gsuite/login/_meta/fields.yml b/x-pack/filebeat/module/gsuite/login/_meta/fields.yml new file mode 100644 index 00000000000..dc8e9711616 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/login/_meta/fields.yml @@ -0,0 +1,21 @@ +- name: login + type: group + fields: + - name: affected_email_address + type: keyword + - name: challenge_method + type: keyword + description: > + Login challenge method. For a list of possible values refer to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login. + - name: failure_type + type: keyword + description: > + Login failure type. For a list of possible values refer to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login. + - name: type + type: keyword + description: > + Login credentials type. For a list of possible values refer to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login. + - name: is_second_factor + type: boolean + - name: is_suspicious + type: boolean diff --git a/x-pack/filebeat/module/gsuite/login/config/config.yml b/x-pack/filebeat/module/gsuite/login/config/config.yml new file mode 100644 index 00000000000..b501012b3d2 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/login/config/config.yml @@ -0,0 +1,50 @@ +{{ if eq .input "httpjson" }} +type: httpjson + +url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user_key }}/applications/login +json_objects_array: items +split_events_by: events + +interval: {{ .interval }} + +{{ if .http_client_timeout }} +http_client_timeout: {{ .http_client_timeout }} +{{ end }} + +oauth2.provider: google +oauth2.google.jwt_file: {{ .jwt_file }} +oauth2.google.delegated_account: {{ .delegated_account }} +oauth2.scopes: + - https://www.googleapis.com/auth/admin.reports.audit.readonly + +date_cursor.url_field: startTime +date_cursor.initial_interval: {{ .initial_interval }} + +pagination.id_field: nextPageToken +pagination.url_field: pageToken + +{{ else if eq .input "file" }} +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 + - script: + lang: javascript + id: gsuite-common + file: ${path.home}/module/gsuite/config/common.js + - script: + lang: javascript + id: gsuite-login + file: ${path.home}/module/gsuite/login/config/pipeline.js diff --git a/x-pack/filebeat/module/gsuite/login/config/pipeline.js b/x-pack/filebeat/module/gsuite/login/config/pipeline.js new file mode 100644 index 00000000000..13c155661a0 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/login/config/pipeline.js @@ -0,0 +1,98 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +var login = (function () { + var processor = require("processor"); + + var categorizeEvent = function(evt) { + evt.Put("event.category", ["authentication"]); + switch (evt.Get("event.action")) { + case "login_failure": + evt.Put("event.type", ["start"]); + evt.Put("event.outcome", "failure"); + break; + case "login_success": + evt.Put("event.type", ["start"]); + evt.Put("event.outcome", "success"); + break; + case "logout": + evt.Put("event.type", ["end"]); + break; + case "account_disabled_generic": + case "account_disabled_spamming_through_relay": + case "account_disabled_spamming": + case "account_disabled_hijacked": + case "account_disabled_password_leak": + evt.Put("event.type", ["user", "change"]); + break; + case "gov_attack_warning": + case "login_challenge": + case "login_verification": + case "suspicious_login": + case "suspicious_login_less_secure_app": + case "suspicious_programmatic_login": + evt.Put("event.type", ["info"]); + break; + } + }; + + var getParamValue = function(param) { + if (param.value) { + return param.value; + } + if (param.multiValue) { + return param.multiValue; + } + }; + + var processParams = function(evt) { + var params = evt.Get("json.events.parameters"); + if (!params || !Array.isArray(params)) { + return; + } + + var prefixRegex = /^(login_)/; + + params.forEach(function(p){ + p.name = p.name.replace(prefixRegex, ""); + switch (p.name) { + // According to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login + // this is a timestamp in microseconds + case "timestamp": + var millis = p.intValue / 1000; + evt.Put("event.start", new Date(millis).toUTCString()); + break; + case "challenge_status": + if (p.value === "Challenge Passed") { + evt.Put("event.outcome", "success"); + } else { + evt.Put("event.outcome", "failure"); + } + break; + case "is_second_factor": + case "is_suspicious": + evt.Put("gsuite.login."+p.name, p.boolValue); + break; + // the rest of params are strings + default: + evt.Put("gsuite.login."+p.name, getParamValue(p)); + } + }); + + evt.Delete("json.events.parameters"); + }; + + var pipeline = new processor.Chain() + .Add(categorizeEvent) + .Add(processParams) + .Build(); + + return { + process: pipeline.Run, + }; +}()); + +function process(evt) { + return login.process(evt); +} diff --git a/x-pack/filebeat/module/gsuite/login/manifest.yml b/x-pack/filebeat/module/gsuite/login/manifest.yml new file mode 100644 index 00000000000..48570efe448 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/login/manifest.yml @@ -0,0 +1,24 @@ +module_version: 1.0 + +var: + - name: input + default: httpjson + - name: jwt_file + - name: delegated_account + - name: initial_interval + default: 24h + - name: http_client_timeout + default: 60s + - name: user_key + default: all + - name: interval + default: 2h + - name: tags + default: [forwarded] + +input: config/config.yml +ingest_pipeline: ../ingest/common.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log b/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log new file mode 100644 index 00000000000..b721c74bf48 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log @@ -0,0 +1,14 @@ +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"account_disabled_password_leak","parameters":[{"name":"affected_email_address","value":"foo@elastic.co"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"suspicious_login","parameters":[{"name":"affected_email_address","value":"foo@elastic.co"},{"name":"login_timestamp","intValue":1593695305123456}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"suspicious_login_less_secure_app","parameters":[{"name":"affected_email_address","value":"foo@elastic.co"},{"name":"login_timestamp","intValue":1593695305123456}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"suspicious_programmatic_login","parameters":[{"name":"affected_email_address","value":"foo@elastic.co"},{"name":"login_timestamp","intValue":1593695305123456}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"account_disabled_generic","parameters":[{"name":"affected_email_address","value":"foo@elastic.co"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"account_disabled_spamming_through_relay","parameters":[{"name":"affected_email_address","value":"foo@elastic.co"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"account_disabled_spamming","parameters":[{"name":"affected_email_address","value":"foo@elastic.co"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"account_disabled_hijacked","parameters":[{"name":"affected_email_address","value":"foo@elastic.co"},{"name":"login_timestamp","intValue":1593695305123456}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"account_warning","name":"gov_attack_warning"}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"login","name":"login_failure","parameters":[{"name":"login_challenge_method","value":"backup_code"},{"name":"login_failure_type","value":"login_failure_access_code_disallowed"},{"name":"login_type","value":"exchange"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"login","name":"login_challenge","parameters":[{"name":"login_challenge_method","value":"backup_code"},{"name":"login_challenge_status","value":"Challenge Passed."},{"name":"login_type","value":"exchange"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"login","name":"login_verification","parameters":[{"name":"is_second_factor","boolValue":false},{"name":"login_challenge_method","value":"backup_code"},{"name":"login_challenge_status","value":"Challenge Passed."},{"name":"login_type","value":"exchange"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"login","name":"logout","parameters":[{"name":"login_type","value":"exchange"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"login","name":"login_success","parameters":[{"name":"login_challenge_method","value":"backup_code"},{"name":"is_suspicious","boolValue":false},{"name":"login_type","value":"exchange"}]}} diff --git a/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log-expected.json b/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log-expected.json new file mode 100644 index 00000000000..91418ce7025 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log-expected.json @@ -0,0 +1,496 @@ +[ + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "account_disabled_password_leak", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"account_disabled_password_leak\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"}]}}", + "event.provider": "login", + "event.type": [ + "user", + "change" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.affected_email_address": "foo@elastic.co", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 0, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "account_disabled_generic", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"account_disabled_generic\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"}]}}", + "event.provider": "login", + "event.type": [ + "user", + "change" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.affected_email_address": "foo@elastic.co", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 1776, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "account_disabled_spamming_through_relay", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"account_disabled_spamming_through_relay\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"}]}}", + "event.provider": "login", + "event.type": [ + "user", + "change" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.affected_email_address": "foo@elastic.co", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 2176, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "account_disabled_spamming", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"account_disabled_spamming\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"}]}}", + "event.provider": "login", + "event.type": [ + "user", + "change" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.affected_email_address": "foo@elastic.co", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 2591, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "gov_attack_warning", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"gov_attack_warning\"}}", + "event.provider": "login", + "event.type": [ + "info" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 3448, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "login_failure", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"login\",\"name\":\"login_failure\",\"parameters\":[{\"name\":\"login_challenge_method\",\"value\":\"backup_code\"},{\"name\":\"login_failure_type\",\"value\":\"login_failure_access_code_disallowed\"},{\"name\":\"login_type\",\"value\":\"exchange\"}]}}", + "event.outcome": "failure", + "event.provider": "login", + "event.type": [ + "start" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "login", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.challenge_method": "backup_code", + "gsuite.login.failure_type": "login_failure_access_code_disallowed", + "gsuite.login.type": "exchange", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 3768, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "login_challenge", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"login\",\"name\":\"login_challenge\",\"parameters\":[{\"name\":\"login_challenge_method\",\"value\":\"backup_code\"},{\"name\":\"login_challenge_status\",\"value\":\"Challenge Passed.\"},{\"name\":\"login_type\",\"value\":\"exchange\"}]}}", + "event.outcome": "failure", + "event.provider": "login", + "event.type": [ + "info" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "login", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.challenge_method": "backup_code", + "gsuite.login.type": "exchange", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 4262, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "login_verification", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"login\",\"name\":\"login_verification\",\"parameters\":[{\"name\":\"is_second_factor\",\"boolValue\":false},{\"name\":\"login_challenge_method\",\"value\":\"backup_code\"},{\"name\":\"login_challenge_status\",\"value\":\"Challenge Passed.\"},{\"name\":\"login_type\",\"value\":\"exchange\"}]}}", + "event.outcome": "failure", + "event.provider": "login", + "event.type": [ + "info" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "login", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.challenge_method": "backup_code", + "gsuite.login.is_second_factor": false, + "gsuite.login.type": "exchange", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 4743, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "logout", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"login\",\"name\":\"logout\",\"parameters\":[{\"name\":\"login_type\",\"value\":\"exchange\"}]}}", + "event.provider": "login", + "event.type": [ + "end" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "login", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.type": "exchange", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 5273, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "login_success", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"login\",\"name\":\"login_success\",\"parameters\":[{\"name\":\"login_challenge_method\",\"value\":\"backup_code\"},{\"name\":\"is_suspicious\",\"boolValue\":false},{\"name\":\"login_type\",\"value\":\"exchange\"}]}}", + "event.outcome": "success", + "event.provider": "login", + "event.type": [ + "start" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "login", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.challenge_method": "backup_code", + "gsuite.login.is_suspicious": false, + "gsuite.login.type": "exchange", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 5627, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/gsuite/saml/config/config.yml b/x-pack/filebeat/module/gsuite/saml/config/config.yml index 4139260481a..1e703737e0d 100644 --- a/x-pack/filebeat/module/gsuite/saml/config/config.yml +++ b/x-pack/filebeat/module/gsuite/saml/config/config.yml @@ -17,10 +17,12 @@ oauth2.google.delegated_account: {{ .delegated_account }} oauth2.scopes: - https://www.googleapis.com/auth/admin.reports.audit.readonly -date_cursor.field: id.time date_cursor.url_field: startTime date_cursor.initial_interval: {{ .initial_interval }} +pagination.id_field: nextPageToken +pagination.url_field: pageToken + {{ else if eq .input "file" }} type: log paths: diff --git a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml index 88afbc0d629..773ab620173 100644 --- a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml +++ b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml @@ -17,10 +17,12 @@ oauth2.google.delegated_account: {{ .delegated_account }} oauth2.scopes: - https://www.googleapis.com/auth/admin.reports.audit.readonly -date_cursor.field: id.time date_cursor.url_field: startTime date_cursor.initial_interval: {{ .initial_interval }} +pagination.id_field: nextPageToken +pagination.url_field: pageToken + {{ else if eq .input "file" }} type: log paths: diff --git a/x-pack/filebeat/modules.d/gsuite.yml.disabled b/x-pack/filebeat/modules.d/gsuite.yml.disabled index efb79107ef4..97a3fdcf94a 100644 --- a/x-pack/filebeat/modules.d/gsuite.yml.disabled +++ b/x-pack/filebeat/modules.d/gsuite.yml.disabled @@ -2,8 +2,27 @@ # Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-gsuite.html - module: gsuite - # All logs saml: enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s user_accounts: enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s + login: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 5s