Xsiam firehose (#259)

* firehose module for xsiam logs ingestion

* Added terraform force unlock

* adjusted iam polcies, and module to main.tf

* Added Cloudwatch error logging

Added config so we can view Cloudwatch logging errors via cloudwatch

* updated module to use secrets manager via data

AWS secrets mananger now store the http endpoint and access_key for the
firehose http endpoint.

* Added server_side_enrytion to firehose

* added logging group resource and permissions for firehose to log to it

* added cloudwatch subscription

* Commit changes made by code formatters

* renamed the logs subscription name to reflect nacs to xsiam

* updated secret version for xsiam endpoint in production and pre-production

* updated secret version for xsiam endpoint in development,production and pre-production

* locked down permissive permissions

---------

Co-authored-by: Jahir <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Makefile

@@ -75,6 +75,10 @@ init-reconfigure: ## terraform init --reconfigure
 init-upgrade: ## terraform init -upgrade
 	$(DOCKER_RUN) /bin/bash -c "terraform init -upgrade --backend-config=\"key=terraform.${ENV}.state\""
 
+.PHONY: unlock
+unlock: ## Terraform unblock (make force-unlock ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)
+	$(DOCKER_RUN) /bin/bash -c "terraform force-unlock ${ID}"
+
 .PHONY: import
 import: ## terraform import e.g. (make import IMPORT_ARGUMENT=module.foo.bar some_resource)
 	$(DOCKER_RUN) /bin/bash -c "terraform import ${IMPORT_ARGUMENT}"

data.tf

@@ -1,3 +1,9 @@
+locals {
+  xaiam_secrets_version_development    = "2e73a1de-af34-4c1d-a8ce-759df5b7bf75"
+  xaiam_secrets_version_pre_production = "9a071db2-4ed2-4c3f-9568-5ef2d5299dc4"
+  xaiam_secrets_version_production     = "a275ae6e-fc4c-4341-bb63-064f4e2fe209"
+}
+
 #-----------------------------------------------------------------
 ### Getting the staff-device-shared-services-infrastructure state
 #-----------------------------------------------------------------
@@ -10,3 +16,12 @@ data "terraform_remote_state" "staff-device-shared-services-infrastructure" {
     region = "eu-west-2"
   }
 }
+
+data "aws_secretsmanager_secret" "xsiam_endpoint_secrets" {
+  name = "/nac-server/${terraform.workspace}/xsiam_endpoint_secrets"
+}
+
+data "aws_secretsmanager_secret_version" "xaiam_secrets_version" {
+  secret_id  = data.aws_secretsmanager_secret.xsiam_endpoint_secrets.id
+  version_id = terraform.workspace == "pre-production" ? local.xaiam_secrets_version_pre_production : terraform.workspace == "production" ? local.xaiam_secrets_version_production : local.xaiam_secrets_version_development
+}

main.tf

@@ -310,3 +310,16 @@ module "performance_testing" {
     aws = aws.env
   }
 }
+
+module "kinesis_firehose_xsiam" {
+  source                                = "./modules/kinesis_firehose_xsiam"
+  http_endpoint                         = jsondecode(data.aws_secretsmanager_secret_version.xaiam_secrets_version.secret_string)["http_endpoint"]
+  access_key                            = jsondecode(data.aws_secretsmanager_secret_version.xaiam_secrets_version.secret_string)["access_key"]
+  prefix                                = "${module.label.id}-xsiam"
+  tags                                  = module.label.tags
+  cloudwatch_log_group_for_subscription = module.radius.cloudwatch.server_log_group_name
+
+  providers = {
+    aws = aws.env
+  }
+}

modules/kinesis_firehose_xsiam/log_group_subscription.tf

@@ -0,0 +1,53 @@
+resource "aws_cloudwatch_log_subscription_filter" "nacs_server_xsiam_subscription" {
+  name            = "xsiam-delivery-stream-${var.prefix}"
+  role_arn        = aws_iam_role.this.arn
+  log_group_name  = var.cloudwatch_log_group_for_subscription
+  filter_pattern  = ""
+  destination_arn = aws_kinesis_firehose_delivery_stream.xsiam_delivery_stream.arn
+}
+
+resource "aws_iam_role" "this" {
+  name_prefix        = var.prefix
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "logs.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_policy" "put_record" {
+  name_prefix = var.prefix
+  tags        = var.tags
+  policy      = <<-EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "firehose:PutRecord",
+                "firehose:PutRecordBatch"
+            ],
+            "Resource": [
+                "${aws_kinesis_firehose_delivery_stream.xsiam_delivery_stream.arn}"
+            ]
+        }
+    ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "this" {
+  role       = aws_iam_role.this.name
+  policy_arn = aws_iam_policy.put_record.arn
+}

modules/kinesis_firehose_xsiam/main.tf

@@ -0,0 +1,134 @@
+resource "aws_kinesis_firehose_delivery_stream" "xsiam_delivery_stream" {
+  name        = "xsiam-delivery-stream-${var.prefix}"
+  destination = "http_endpoint"
+
+  server_side_encryption {
+    enabled = true
+  }
+
+  http_endpoint_configuration {
+    url                = var.http_endpoint
+    name               = var.prefix
+    access_key         = var.access_key
+    buffering_size     = 5
+    buffering_interval = 300
+    role_arn           = aws_iam_role.xsiam_kinesis_firehose_role.arn
+    s3_backup_mode     = "FailedDataOnly"
+
+    cloudwatch_logging_options {
+      enabled         = true
+      log_group_name  = aws_cloudwatch_log_group.xsiam_delivery_group.name
+      log_stream_name = aws_cloudwatch_log_stream.xsiam_delivery_stream.name
+    }
+  }
+
+  s3_configuration {
+    role_arn           = aws_iam_role.xsiam_kinesis_firehose_role.arn
+    bucket_arn         = aws_s3_bucket.xsiam_firehose_bucket.arn
+    buffer_size        = 10
+    buffer_interval    = 400
+    compression_format = "GZIP"
+  }
+}
+
+resource "aws_cloudwatch_log_group" "xsiam_delivery_group" {
+  name = "xsiam-delivery-stream-${var.prefix}"
+
+  retention_in_days = 90
+}
+
+resource "aws_cloudwatch_log_stream" "xsiam_delivery_stream" {
+  name           = "errors"
+  log_group_name = aws_cloudwatch_log_group.xsiam_delivery_group.name
+}
+
+resource "aws_iam_role" "xsiam_kinesis_firehose_role" {
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "firehose.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "xsiam_kinesis_firehose_role_policy" {
+  role = aws_iam_role.xsiam_kinesis_firehose_role.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "logs:DescribeLogGroups",
+          "logs:DescribeLogStreams",
+          "logs:GetLogEvents"
+        ]
+        Effect   = "Allow"
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "kinesis_firehose_error_log_role_attachment" {
+  policy_arn = aws_iam_policy.xsiam_kinesis_firehose_error_log_policy.arn
+  role       = aws_iam_role.xsiam_kinesis_firehose_role.name
+
+}
+
+resource "aws_iam_policy" "xsiam_kinesis_firehose_error_log_policy" {
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "logs:PutLogEvents",
+        ]
+        Effect = "Allow"
+        Resource = [
+          "${aws_cloudwatch_log_group.xsiam_delivery_group.arn}/*"
+        ]
+      }
+    ]
+  })
+}
+
+
+resource "aws_iam_role_policy_attachment" "kinesis_role_attachment" {
+  policy_arn = aws_iam_policy.s3_kinesis_xsiam_policy.arn
+  role       = aws_iam_role.xsiam_kinesis_firehose_role.name
+
+}
+
+resource "aws_iam_policy" "s3_kinesis_xsiam_policy" {
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "s3:AbortMultipartUpload",
+          "s3:GetBucketLocation",
+          "s3:GetObject",
+          "s3:ListBucket",
+          "s3:ListBucketMultipartUploads",
+          "s3:PutObject"
+        ]
+        Effect = "Allow"
+        Resource = [
+          aws_s3_bucket.xsiam_firehose_bucket.arn,
+          "${aws_s3_bucket.xsiam_firehose_bucket.arn}/*"
+        ]
+      }
+    ]
+  })
+}

modules/kinesis_firehose_xsiam/required_providers.tf

@@ -0,0 +1,6 @@
+terraform {
+  required_providers {
+    aws = {
+    }
+  }
+}

modules/kinesis_firehose_xsiam/s3.tf

@@ -0,0 +1,5 @@
+resource "aws_s3_bucket" "xsiam_firehose_bucket" {
+  bucket = "xsiam-firehose-${var.prefix}"
+
+  tags = var.tags
+}

modules/kinesis_firehose_xsiam/variable.tf

@@ -0,0 +1,15 @@
+variable "http_endpoint" {
+  type = string
+}
+variable "prefix" {
+  type = string
+}
+variable "access_key" {
+  type = string
+}
+variable "tags" {
+  type = map(string)
+}
+variable "cloudwatch_log_group_for_subscription" {
+  type = string
+}