feat: Support custom log configuration #342

oponomarov-tu · 2023-04-20T12:38:06Z

Description

Add an input variable to allow us to override the logConfiguration stanza for the ECS task for Atlantis.
Use count instead of for_each for iam policy attachments.

Motivation and Context

Custom log configuration is needed when, for example, running a sidecar container to ship logs to Elasticsearch / Humio / etc.
Fix chicken-and-egg problem when running terraform plan for policies yet to be created (this will recreate policy attachments with indexes instead of keys).

Breaking Changes

resource "aws_iam_role_policy_attachment" "ecs_task_execution" will be recreated due to the nature of using index-based (count) vs. map-name-based (for_each) approach.

How Has This Been Tested?

I have updated at least one of the examples/* to demonstrate and validate my change(s)
I have tested and validated these changes using one or more of the provided examples/* projects

I have executed pre-commit run -a on my pull request

The changes have been tested with the latest version of module running in production. The recreated policy attachments had no influence on the running Atlantis instance.

Snippets:

data "aws_iam_policy_document" "ecs_cloudwatch" {
  version = "2012-10-17"
  statement {
    actions = [
      "logs:CreateLogGroup",
    ]
    resources = ["*"]
  }
}

resource "aws_iam_policy" "ecs_cloudwatch" {
  name        = "atlantis-ecs-cloudwatch-policy"
  path        = "/"
  description = "Allows Atlantis to create CloudWatch Log Groups"
  policy      = data.aws_iam_policy_document.ecs_cloudwatch.json
}

module "container_definition_fluentbit" {
  source  = "cloudposse/ecs-container-definition/aws"
  version = "v0.58.1"

  container_name  = "log-router"
  container_image = "amazon/aws-for-fluent-bit:stable"
  essential       = true

  firelens_configuration = {
    type = "fluentbit"

    options = {
      enable-ecs-log-metadata = "true",
      config-file-type        = "file",
      config-file-value       = "/fluent-bit/configs/parse-json.conf",
    }
  }

  log_configuration = {
    logDriver = "awslogs",
    options = {
      awslogs-create-group  = "true",
      awslogs-group         = "atlantis-humio",
      awslogs-region        = var.region,
      awslogs-stream-prefix = "humio-ingester",
    }
  }
}

module "atlantis" {
  source  = "terraform-aws-modules/atlantis/aws"
  version = "3.26.0"
  ...
  # IAM policies
  policies_arn = [
    "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
    ...
    aws_iam_policy.ecs_cloudwatch.arn,
  ]
  ... 
  # Extra container definitions
  extra_container_definitions = [module.container_definition_fluentbit.json_map_object]

  custom_log_configuration = {
    logDriver = "awsfirelens",
    options = {
      "Name" : "es",
      "Host" : "cloud.humio.com",
      "Port" : "9200",
      "HTTP_User" : "defaultRepoName",
      "Logstash_Format" : "On",
      "Type" : "flb_type",
      "Time_Key" : "@timestamp",
      "Replace_Dots" : "On",
      "Logstash_Prefix" : "atlantis",
      "Buffer_Size" : "5MB",
      "tls" : "On",
    }
    secretOptions = [
      {
        name      = "HTTP_Passwd",
        valueFrom = data.aws_ssm_parameter.humio_ingest_token.name,
      }
    ]
  }

github-actions · 2023-05-21T00:10:59Z