diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index d0b96ad1be..984b62872f 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -256,6 +256,12 @@ jobs: aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} + - id: azure-auth + name: Azure login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - id: google-auth name: Authenticate to Google Cloud uses: google-github-actions/auth@v2 @@ -263,12 +269,6 @@ jobs: workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} - - id: azure-auth - name: Azure login - uses: azure/login@v2 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - - name: Set TF_STATE_FOLDER run: | echo "TF_STATE_FOLDER=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_ENV diff --git a/deploy/cloud/modules/gcp/vm/main.tf b/deploy/cloud/modules/gcp/vm/main.tf new file mode 100644 index 0000000000..7026bb65a0 --- /dev/null +++ b/deploy/cloud/modules/gcp/vm/main.tf @@ -0,0 +1,49 @@ +resource "random_id" "id" { + byte_length = 4 +} + +locals { + vm_private_key_file = "${path.module}/gcp-vm-${random_id.id.hex}.pem" + vm_username = "ubuntu" + deploy_name = "${var.deployment_name}-${random_id.id.hex}" +} + +resource "tls_private_key" "gcp_vm_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "local_file" "ssh_private_key" { + filename = local.vm_private_key_file + content = tls_private_key.gcp_vm_key.private_key_pem + file_permission = 0400 +} + +resource "google_compute_instance" "vm_instance" { + name = local.deploy_name + machine_type = var.machine_type + zone = var.zone + labels = var.specific_tags + + boot_disk { + initialize_params { + image = var.disk_image + } + } + + network_interface { + network = var.network + + access_config { + // Ephemeral public IP + } + } + + metadata = { + ssh-keys = "${local.vm_username}:${tls_private_key.gcp_vm_key.public_key_openssh}" + } + + service_account { + scopes = var.scopes + } +} diff --git a/deploy/cloud/modules/gcp/vm/outputs.tf b/deploy/cloud/modules/gcp/vm/outputs.tf new file mode 100644 index 0000000000..7342dcc43b --- /dev/null +++ b/deploy/cloud/modules/gcp/vm/outputs.tf @@ -0,0 +1,14 @@ +output "gcp_vm_puglic_ip" { + description = "GCP VM instance public IP" + value = google_compute_instance.vm_instance.network_interface[0].access_config[0].nat_ip +} + +output "gcp_vm_ssh_cmd" { + description = "Use this command to SSH into the GCP VM instance" + value = "ssh -i ${local.vm_private_key_file} ${local.vm_username}@${google_compute_instance.vm_instance.network_interface[0].access_config[0].nat_ip}" +} + +output "gcp_vm_ssh_key" { + description = "The path to the private SSH key file." + value = local.vm_private_key_file +} diff --git a/deploy/cloud/modules/gcp/vm/terraform.tf b/deploy/cloud/modules/gcp/vm/terraform.tf new file mode 100644 index 0000000000..53083b1855 --- /dev/null +++ b/deploy/cloud/modules/gcp/vm/terraform.tf @@ -0,0 +1,11 @@ +terraform { + required_providers { + + google = { + source = "hashicorp/google" + version = ">= 5.0.0" + } + } + + required_version = ">= 1.3, <2.0.0" +} diff --git a/deploy/cloud/modules/gcp/vm/variables.tf b/deploy/cloud/modules/gcp/vm/variables.tf new file mode 100644 index 0000000000..3254807ffa --- /dev/null +++ b/deploy/cloud/modules/gcp/vm/variables.tf @@ -0,0 +1,39 @@ +variable "deployment_name" { + description = "The base name used for resources in this deployment." + type = string +} + +variable "specific_tags" { + description = "List of tags in the format 'key=value'" + type = map(string) + default = {} +} + +variable "machine_type" { + description = "The machine type to use for the VM." + type = string + default = "n2-standard-4" +} + +variable "disk_image" { + description = "The image to use for the VM's boot disk." + type = string + default = "ubuntu-os-cloud/ubuntu-2204-lts" +} + +variable "zone" { + description = "The GCP zone where the VM will be deployed." + type = string + default = "us-central1-a" +} + +variable "network" { + description = "The network to attach the VM to." + type = string +} + +variable "scopes" { + description = "The scopes to attach to the service account." + type = list(string) + default = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloudplatformorganizations"] +} diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf index 6dd89b2895..d2bf0f7473 100644 --- a/deploy/test-environments/main.tf +++ b/deploy/test-environments/main.tf @@ -2,6 +2,10 @@ provider "aws" { region = var.region } +provider "google" { + project = var.gcp_project_id +} + provider "azurerm" { features {} } @@ -15,6 +19,7 @@ locals { owner = "${var.owner}" deployment = "${var.deployment_name}" } + ec_url = "https://cloud.elastic.co" ec_headers = { Content-type = "application/json" @@ -42,6 +47,17 @@ module "aws_ec2_for_cspm" { specific_tags = merge(local.common_tags, { "ec2_type" : "cspm" }) } +module "gcp_audit_logs" { + count = var.cdr_infra ? 1 : 0 + providers = { google : google } + source = "../cloud/modules/gcp/vm" + + deployment_name = var.deployment_name + network = "default" + specific_tags = merge(local.common_tags, { "vm_instance" : "audit-logs" }) + +} + resource "random_string" "suffix" { length = 3 special = false diff --git a/deploy/test-environments/output.tf b/deploy/test-environments/output.tf index 8467cd2bac..a0ca08c076 100644 --- a/deploy/test-environments/output.tf +++ b/deploy/test-environments/output.tf @@ -52,6 +52,21 @@ output "ec2_cloudtrail_key" { sensitive = true } +output "gcp_audit_logs_ssh_cmd" { + value = var.cdr_infra ? module.gcp_audit_logs[0].gcp_vm_ssh_cmd : null + # sensitive = true +} + +output "gcp_audit_logs_public_ip" { + value = var.cdr_infra ? module.gcp_audit_logs[0].gcp_vm_puglic_ip : null + # sensitive = true +} + +output "gcp_audit_logs_key" { + value = var.cdr_infra ? module.gcp_audit_logs[0].gcp_vm_ssh_key : null + # sensitive = true +} + output "az_vm_activity_logs_ssh_cmd" { value = var.cdr_infra ? module.azure_vm_activity_logs[0].azure_vm_ssh_cmd : null sensitive = true @@ -66,7 +81,6 @@ output "az_vm_activity_logs_key" { value = var.cdr_infra ? module.azure_vm_activity_logs[0].azure_vm_ssh_key : null sensitive = true } - # ============================================================= # Elastic Cloud output diff --git a/deploy/test-environments/terraform.tf b/deploy/test-environments/terraform.tf index 600e483785..42f07e302e 100644 --- a/deploy/test-environments/terraform.tf +++ b/deploy/test-environments/terraform.tf @@ -5,6 +5,11 @@ terraform { version = "~> 4.15.0" } + google = { + source = "hashicorp/google" + version = ">= 5.0.0" + } + azurerm = { source = "hashicorp/azurerm" version = ">= 3.11, < 4.0" diff --git a/deploy/test-environments/variables.tf b/deploy/test-environments/variables.tf index ffad9a5595..abb0d398c4 100644 --- a/deploy/test-environments/variables.tf +++ b/deploy/test-environments/variables.tf @@ -23,6 +23,13 @@ variable "ami_map" { } } +# GCP project ID +variable "gcp_project_id" { + description = "GCP project ID" + type = string + default = "default" +} + # Elastic Cloud variables # =========================================== variable "ec_api_key" { diff --git a/tests/integrations_setup/configuration_fleet.py b/tests/integrations_setup/configuration_fleet.py index d808e96709..ceea092ec5 100644 --- a/tests/integrations_setup/configuration_fleet.py +++ b/tests/integrations_setup/configuration_fleet.py @@ -54,6 +54,12 @@ gcp_dm_config.credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS", "") gcp_dm_config.service_account_json_path = os.getenv("SERVICE_ACCOUNT_JSON_PATH", "") +gcp_audit_config = Munch() +gcp_audit_config.credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS", "") +gcp_audit_config.project_id = os.getenv("GOOGLE_CLOUD_PROJECT", "") +gcp_audit_config.topic_name = os.getenv("GCP_TOPIC_NAME", "") +gcp_audit_config.subscription_name = os.getenv("GCP_SUBSCRIPTION_NAME", "") + # Used for Azure deployment on stack 8.11.* (1.6.* package version) azure_arm_parameters = Munch() azure_arm_parameters.deployment_name = os.getenv("DEPLOYMENT_NAME", "") diff --git a/tests/integrations_setup/data/gcp-audit-logs-pkg.json b/tests/integrations_setup/data/gcp-audit-logs-pkg.json new file mode 100644 index 0000000000..d5ae17d289 --- /dev/null +++ b/tests/integrations_setup/data/gcp-audit-logs-pkg.json @@ -0,0 +1,265 @@ +{ + "package": { + "name": "gcp", + "version": "" + }, + "name": "", + "namespace": "", + "description": "", + "policy_ids": [""], + "vars": { + "project_id": "", + "credentials_file": "/home/ubuntu/credentials.json" + }, + "inputs": { + "audit-gcp-pubsub": { + "enabled": true, + "streams": { + "gcp.audit": { + "enabled": true, + "vars": { + "topic": "", + "subscription_name": "", + "subscription_create": false, + "tags": [ + "forwarded", + "gcp-audit" + ], + "preserve_original_event": true, + "keep_json": false + } + } + } + }, + "firewall-gcp-pubsub": { + "enabled": false, + "streams": { + "gcp.firewall": { + "enabled": true, + "vars": { + "topic": "cloud-logging-firewall", + "subscription_name": "filebeat-gcp-firewall", + "subscription_create": false, + "tags": [ + "forwarded", + "gcp-firewall" + ], + "preserve_original_event": false, + "keep_json": false + } + } + } + }, + "vpcflow-gcp-pubsub": { + "enabled": false, + "streams": { + "gcp.vpcflow": { + "enabled": true, + "vars": { + "topic": "cloud-logging-vpcflow", + "subscription_name": "filebeat-gcp-vpcflow", + "subscription_create": false, + "tags": [ + "forwarded", + "gcp-vpcflow" + ], + "preserve_original_event": false, + "keep_json": false + } + } + } + }, + "dns-gcp-pubsub": { + "enabled": false, + "streams": { + "gcp.dns": { + "enabled": true, + "vars": { + "topic": "cloud-logging-dns", + "subscription_name": "filebeat-gcp-dns", + "subscription_create": false, + "tags": [ + "forwarded", + "gcp-dns" + ], + "preserve_original_event": false, + "keep_json": false + } + } + } + }, + "billing-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.billing": { + "enabled": true, + "vars": { + "period": "24h", + "table_pattern": "gcp_billing_export_v1", + "cost_type": "regular" + } + } + } + }, + "compute-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.compute": { + "enabled": true, + "vars": { + "regions": [], + "period": "60s", + "tags": [ + "gcp-compute" + ] + } + } + } + }, + "firestore-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.firestore": { + "enabled": true, + "vars": { + "regions": [], + "period": "60s", + "tags": [ + "gcp-firestore" + ] + } + } + } + }, + "loadbalancing-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.loadbalancing_metrics": { + "enabled": true, + "vars": { + "regions": [], + "period": "60s" + } + } + } + }, + "loadbalancing-gcp-pubsub": { + "enabled": false, + "streams": { + "gcp.loadbalancing_logs": { + "enabled": true, + "vars": { + "topic": "cloud-logging-load_balancer", + "subscription_name": "filebeat-gcp-load_balancer", + "subscription_create": false, + "tags": [ + "forwarded", + "gcp-loadbalancing_logs" + ], + "preserve_original_event": false + } + } + } + }, + "storage-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.storage": { + "enabled": true, + "vars": { + "regions": [], + "period": "5m", + "tags": [ + "gcp-storage" + ] + } + } + } + }, + "gke-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.gke": { + "enabled": true, + "vars": { + "regions": [], + "period": "60s" + } + } + } + }, + "dataproc-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.dataproc": { + "enabled": true, + "vars": { + "regions": [], + "period": "60s" + } + } + } + }, + "pubsub-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.pubsub": { + "enabled": true, + "vars": { + "period": "60s" + } + } + } + }, + "redis-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.redis": { + "enabled": true, + "vars": { + "regions": [], + "period": "60s" + } + } + } + }, + "cloudrun-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.cloudrun_metrics": { + "enabled": true, + "vars": { + "regions": [], + "period": "60s" + } + } + } + }, + "cloudsql-gcp/metrics": { + "enabled": false, + "streams": { + "gcp.cloudsql_mysql": { + "enabled": true, + "vars": { + "period": "60s" + } + }, + "gcp.cloudsql_postgresql": { + "enabled": true, + "vars": { + "period": "60s", + "tags": [ + "gcp-cloudsql-postgresql" + ] + } + }, + "gcp.cloudsql_sqlserver": { + "enabled": true, + "vars": { + "period": "60s" + } + } + } + } + } +} diff --git a/tests/integrations_setup/install_gcp_audit_logs_integration.py b/tests/integrations_setup/install_gcp_audit_logs_integration.py new file mode 100755 index 0000000000..96a6065e63 --- /dev/null +++ b/tests/integrations_setup/install_gcp_audit_logs_integration.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +""" +This script installs Audit Logs GCP integration + +The following steps are performed: +1. Create an agent policy. +2. Create an Audit Logs integration. +3. Create an Audit Logs bash script to be deployed on a host. +""" +from pathlib import Path + +import configuration_fleet as cnfg +from fleet_api.agent_policy_api import create_agent_policy +from fleet_api.common_api import ( + get_artifact_server, + get_enrollment_token, + get_fleet_server_host, + get_package_version, +) +from fleet_api.package_policy_api import create_integration +from fleet_api.utils import read_json, render_template, update_key_value +from loguru import logger +from munch import Munch +from package_policy import SIMPLIFIED_AGENT_POLICY, generate_random_name +from state_file_manager import HostType, PolicyState, state_manager + +CSPM_EXPECTED_AGENTS = 0 +INTEGRATION_NAME = "AUDIT LOGS GCP" + +agent_launcher_template = Path(__file__).parent / "data/cspm-linux.j2" + +if __name__ == "__main__": + # pylint: disable=duplicate-code + package_version = get_package_version(cfg=cnfg.elk_config, package_name="gcp", prerelease=False) + logger.info(f"Package version: {package_version}") + + logger.info(f"Starting installation of {INTEGRATION_NAME} integration.") + agent_data = SIMPLIFIED_AGENT_POLICY + agent_data["name"] = generate_random_name("audit-logs-gcp") + + package_data = read_json(Path(__file__).parent / "data/gcp-audit-logs-pkg.json") + package_data["name"] = generate_random_name("pkg-audit-logs-gcp") + package_data["package"]["version"] = package_version + package_data["vars"]["project_id"] = cnfg.gcp_audit_config.project_id + + update_key_value( + data=package_data["inputs"]["audit-gcp-pubsub"], + search_key="topic", + value_to_apply=cnfg.gcp_audit_config.topic_name, + ) + + update_key_value( + data=package_data["inputs"]["audit-gcp-pubsub"], + search_key="subscription_name", + value_to_apply=cnfg.gcp_audit_config.subscription_name, + ) + + logger.info("Create agent policy") + agent_policy_id = create_agent_policy(cfg=cnfg.elk_config, json_policy=agent_data) + + logger.info(f"Create {INTEGRATION_NAME} integration") + package_policy_id = create_integration( + cfg=cnfg.elk_config, + pkg_policy=package_data, + agent_policy_id=agent_policy_id, + data={}, + ) + + state_manager.add_policy( + PolicyState( + agent_policy_id, + package_policy_id, + CSPM_EXPECTED_AGENTS, + [], + HostType.LINUX_TAR.value, + package_data["name"], + ), + ) + + manifest_params = Munch() + manifest_params.enrollment_token = get_enrollment_token( + cfg=cnfg.elk_config, + policy_id=agent_policy_id, + ) + + manifest_params.fleet_url = get_fleet_server_host(cfg=cnfg.elk_config) + manifest_params.file_path = Path(__file__).parent / "gcp_audit_logs.sh" + manifest_params.agent_version = cnfg.elk_config.stack_version + manifest_params.artifacts_url = get_artifact_server(cnfg.elk_config.stack_version) + + # Render the template and get the replaced content + rendered_content = render_template(agent_launcher_template, manifest_params.toDict()) + + logger.info(f"Creating {INTEGRATION_NAME} linux manifest") + # Write the rendered content to a file + with open(Path(__file__).parent / "gcp_audit_logs.sh", "w", encoding="utf-8") as agent_launcher_file: + agent_launcher_file.write(rendered_content) + + logger.info(f"Installation of {INTEGRATION_NAME} integration is done")