Skip to content

Commit

Permalink
Check types in googlecloud/audit field renames
Browse files Browse the repository at this point in the history 
Specify type to the convert processor so that fields are renamed only
if the value is (or can be coerced to) the desired type. This prevents the
module from writing incompatible data types into fields and prevents
mapping exceptions.

The data in the google cloud audit logs can vary based on the source service
so any time the module makes an assumption about a field that's not explicitly
documented the module should be very defensive.

The request/response object values would be good candidates for the flattened data type
if we want to make a future change.

Fixes #18465
  • Loading branch information
@andrewkroh
andrewkroh committed Aug 6, 2020
1 parent 5fdcb1c commit 9a7bfce
Show file tree
Hide file tree
Showing 3 changed files with 287 additions and 57 deletions.
240 changes: 183 additions & 57 deletions x-pack/filebeat/module/googlecloud/audit/config/pipeline.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,24 @@ function Audit(keep_original_message) {
ignore_missing: true
});

// Use the monitored resource type's labels to set the cloud metadata.
// The labels can vary based on the resource.type.
// https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource
var setCloudMetadata = new processor.Convert({
fields: [
{from: "json.resource.labels.project_id", to: "cloud.project.id"},
{
from: "json.resource.labels.project_id",
to: "cloud.project.id",
type: "string"
},
{
from: "json.resource.labels.instance_id",
to: "cloud.instance.id",
type: "string"
}
],
ignore_missing: true
ignore_missing: true,
fail_on_error: false,
});

// The log includes a protoPayload field.
Expand All @@ -61,80 +74,195 @@ function Audit(keep_original_message) {

// The LogEntry's protoPayload is moved to the json field. The protoPayload
// contains the structured audit log fields.
// https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog
var convertProtoPayload = new processor.Convert({
fields: [
{from: "json.@type", to: "googlecloud.audit.type"},

{from: "json.authenticationInfo.principalEmail", to: "json.authenticationInfo.principal_email"},
{from: "json.authenticationInfo.authoritySelector", to: "json.authenticationInfo.authority_selector"},
{from: "json.authenticationInfo", to: "googlecloud.audit.authentication_info"},

{from: "json.authorizationInfo", to: "googlecloud.audit.authorization_info"},

{from: "json.methodName", to: "googlecloud.audit.method_name"},

{from: "json.numResponseItems", to: "googlecloud.audit.num_response_items", type: "long"},

{from: "json.request.@type", to: "googlecloud.audit.request.proto_name"},
{from: "json.request.filter", to: "googlecloud.audit.request.filter"},
{from: "json.request.name", to: "googlecloud.audit.request.name"},
{from: "json.request.resourceName", to: "googlecloud.audit.request.resource_name"},

{from: "json.requestMetadata.callerIp", to: "json.requestMetadata.caller_ip", type: "ip"},
{from: "json.requestMetadata.callerSuppliedUserAgent", to: "json.requestMetadata.caller_supplied_user_agent"},
{from: "json.requestMetadata", to: "googlecloud.audit.request_metadata"},

{from: "json.response.@type", to: "googlecloud.audit.response.proto_name"},
{from: "json.response.status", to: "googlecloud.audit.response.status"},
{from: "json.response.details.group", to: "googlecloud.audit.response.details.group"},
{from: "json.response.details.kind", to: "googlecloud.audit.response.details.kind"},
{from: "json.response.details.name", to: "googlecloud.audit.response.details.name"},
{from: "json.response.details.uid", to: "googlecloud.audit.response.details.uid"},

{from: "json.resourceName", to: "googlecloud.audit.resource_name"},

{from: "json.resourceLocation.currentLocations", to: "json.resourceLocation.current_locations"},
{from: "json.resourceLocation", to: "googlecloud.audit.resource_location"},

{from: "json.serviceName", to: "googlecloud.audit.service_name"},

{from: "json.status", to: "googlecloud.audit.status"},

{
from: "json.@type",
to: "googlecloud.audit.type",
type: "string"
},
{
from: "json.authenticationInfo.principalEmail",
to: "googlecloud.audit.authentication_info.principal_email",
type: "string"
},
{
from: "json.authenticationInfo.authoritySelector",
to: "googlecloud.audit.authentication_info.authority_selector",
type: "string"
},
{
from: "json.authorizationInfo",
to: "googlecloud.audit.authorization_info"
// Type is an array of objects.
},
{
from: "json.methodName",
to: "googlecloud.audit.method_name",
type: "string",
},
{
from: "json.numResponseItems",
to: "googlecloud.audit.num_response_items",
type: "long"
},
{
from: "json.request.@type",
to: "googlecloud.audit.request.proto_name",
type: "string"
},
// The values in the request object will depend on the proto type.
// So be very careful about making any assumptions about data shape.
{
from: "json.request.filter",
to: "googlecloud.audit.request.filter",
type: "string"
},
{
from: "json.request.name",
to: "googlecloud.audit.request.name",
type: "string"
},
{
from: "json.request.resourceName",
to: "googlecloud.audit.request.resource_name",
type: "string"
},
{
from: "json.requestMetadata.callerIp",
to: "googlecloud.audit.request_metadata.caller_ip",
type: "ip"
},
{
from: "json.requestMetadata.callerSuppliedUserAgent",
to: "googlecloud.audit.request_metadata.caller_supplied_user_agent",
type: "string",
},
{
from: "json.response.@type",
to: "googlecloud.audit.response.proto_name",
type: "string"
},
// The values in the response object will depend on the proto type.
// So be very careful about making any assumptions about data shape.
{
from: "json.response.status",
to: "googlecloud.audit.response.status",
type: "string"
},
{
from: "json.response.details.group",
to: "googlecloud.audit.response.details.group",
type: "string"
},
{
from: "json.response.details.kind",
to: "googlecloud.audit.response.details.kind",
type: "string"
},
{
from: "json.response.details.name",
to: "googlecloud.audit.response.details.name",
type: "string"
},
{
from: "json.response.details.uid",
to: "googlecloud.audit.response.details.uid",
type: "string",
},
{
from: "json.resourceName",
to: "googlecloud.audit.resource_name",
type: "string",
},
{
from: "json.resourceLocation.currentLocations",
to: "googlecloud.audit.resource_location.current_locations"
// Type is a string array.
},
{
from: "json.serviceName",
to: "googlecloud.audit.service_name",
type: "string",
},
{
from: "json.status.code",
to: "googlecloud.audit.status.code",
type: "integer",
},
{
from: "json.status.message",
to: "googlecloud.audit.status.message",
type: "string"
},
],
mode: "rename",
ignore_missing: true,
fail_on_error: false,
});

// Copy some fields
var copyFields = new processor.Convert({
fields: [
{from: "googlecloud.audit.request_metadata.caller_ip", to: "source.ip"},
{from: "googlecloud.audit.authentication_info.principal_email", to: "user.email"},
{from: "googlecloud.audit.service_name", to: "service.name"},
{from: "googlecloud.audit.request_metadata.caller_supplied_user_agent", to: "user_agent.original"},
{from: "googlecloud.audit.method_name", to: "event.action"},
{
from: "googlecloud.audit.request_metadata.caller_ip",
to: "source.ip",
type: "ip"
},
{
from: "googlecloud.audit.authentication_info.principal_email",
to: "user.email",
type: "string"
},
{
from: "googlecloud.audit.service_name",
to: "service.name",
type: "string"
},
{
from: "googlecloud.audit.request_metadata.caller_supplied_user_agent",
to: "user_agent.original",
type: "string"
},
{
from: "googlecloud.audit.method_name",
to: "event.action",
type: "string"
},
],
ignore_missing: true,
fail_on_error: false,
});

// Drop extra fields
var dropExtraFields = function(evt) {
evt.Delete("json");
evt.Delete("googlecloud.audit.request_metadata.requestAttributes");
evt.Delete("googlecloud.audit.request_metadata.destinationAttributes");
};

// Rename nested fields
var RenameNestedFields = function(evt) {
// Rename nested fields.
var renameNestedFields = function(evt) {
var arr = evt.Get("googlecloud.audit.authorization_info");
for (var i = 0; i < arr.length; i++) {
arr[i].resource_attributes = arr[i].resourceAttributes;
delete arr[i].resourceAttributes;
if (arr[i].resourceAttributes) {
// Convert to snake_case.
arr[i].resource_attributes = arr[i].resourceAttributes;
delete arr[i].resourceAttributes;
}
}
};

// Set ECS categorization fields.
var setECSCategorization = function(evt) {
evt.Put("event.kind", "event");

// google.rpc.Code value for OK is 0.
if (evt.Get("googlecloud.audit.status.code") === 0) {
evt.Put("event.outcome", "success");
return;
}

// Try to use authorization_info.granted when there was no status code.
if (evt.Get("googlecloud.audit.status.code") == null) {
var authorization_info = evt.Get("googlecloud.audit.authorization_info");
if (authorization_info.length === 1) {
Expand All @@ -147,13 +275,11 @@ function Audit(keep_original_message) {
}
} else {
evt.Put("event.outcome", "unknown");
}
} else if (evt.Get("googlecloud.audit.status.code") === 0) {
evt.Put("event.outcome", "success");
} else {
evt.Put("event.outcome", "failure");
}
return;
}
evt.Put("event.kind", "event");

evt.Put("event.outcome", "failure");
};

var pipeline = new processor.Chain()
Expand All @@ -167,7 +293,7 @@ function Audit(keep_original_message) {
.Add(convertProtoPayload)
.Add(copyFields)
.Add(dropExtraFields)
.Add(RenameNestedFields)
.Add(renameNestedFields)
.Add(setECSCategorization)
.Build();

Expand Down
Loading

0 comments on commit 9a7bfce

Please sign in to comment.