diff --git a/.github/actions/aws-asset-inventory-ci/action.yml b/.github/actions/aws-asset-inventory-ci/action.yml index c222b14de7..e4b16234c0 100644 --- a/.github/actions/aws-asset-inventory-ci/action.yml +++ b/.github/actions/aws-asset-inventory-ci/action.yml @@ -37,7 +37,7 @@ runs: AWS_ACCOUNT_TYPE: ${{ inputs.aws-account-type }} shell: bash run: | - ./cloudbeat -c deploy/aws-asset-inventory/cloudbeat-aws-asset-inventory.yml -d '*' & + ./cloudbeat -c deploy/asset-inventory/cloudbeat-aws-asset-inventory.yml -d '*' & - name: Wait for cloudbeat to send some events shell: bash diff --git a/.github/actions/gcp-asset-inventory-ci/action.yml b/.github/actions/gcp-asset-inventory-ci/action.yml new file mode 100644 index 0000000000..6f29660656 --- /dev/null +++ b/.github/actions/gcp-asset-inventory-ci/action.yml @@ -0,0 +1,65 @@ +name: "GCP Asset Inventory CI" +description: "GCP Asset Inventory integration tests" +inputs: + elk-version: + description: "ELK version" + required: true + credentials-json: + description: "GCP Service account key JSON" + required: true + project-id: + description: "GCP Project ID" + required: true + + debug: + description: "debug" + required: false + default: "false" +runs: + using: composite + steps: + - name: Init Integration + uses: ./.github/actions/init-integration + with: + elk-version: ${{ inputs.elk-version }} + + - name: Run cloudbeat in background + env: + ES_HOST: http://localhost:9200 + ES_USERNAME: elastic + ES_PASSWORD: changeme + GCP_PROJECT_ID: ${{ inputs.project-id}} + GCP_CREDENTIALS_JSON: ${{ inputs.credentials-json }} + GCP_ACCOUNT_TYPE: single-account + shell: bash + run: | + ./cloudbeat -c deploy/asset-inventory/cloudbeat-gcp-asset-inventory.yml -d '*' & + + - name: Wait for cloudbeat to send some events + shell: bash + run: sleep 20 + + - name: Check for assets + working-directory: ./tests + env: + USE_K8S: "false" + shell: bash + run: poetry run pytest -k "asset_inventory_gcp" --alluredir=./allure/results/ --clean-alluredir + + - name: Upload test results + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: allure-results-ci-gcp-asset-inventory + path: tests/allure/results/ + overwrite: true + + - if: ${{ failure() || cancelled() || inputs.debug == 'true' }} + name: Upload cloudbeat logs + uses: actions/upload-artifact@v4 + with: + name: cloubeat-logs-ci-gcp-asset-inventory + path: logs/ + if-no-files-found: warn + retention-days: 1 + overwrite: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2222b7f133..e4c3431792 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,28 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_TEST_ACC }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_TEST_ACC }} + ci-gcp-asset-inventory: + needs: [init-hermit] + name: GCP Asset Inventory CI + runs-on: ubuntu-22.04 + timeout-minutes: 60 + permissions: + contents: "read" + id-token: "write" + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Hermit Environment + uses: ./.github/actions/hermit + + - name: Run GCP Asset Inventory integration tests + uses: ./.github/actions/gcp-asset-inventory-ci + with: + elk-version: ${{ env.ELK_VERSION }} + credentials-json: ${{ secrets.GCP_ASSETS_INVENTORY_CREDENTIALS_JSON }} + project-id: "elastic-security-test" + ci-gcp: needs: [ init-hermit ] name: CIS GCP CI diff --git a/deploy/aws-asset-inventory/cloudbeat-aws-asset-inventory.yml b/deploy/asset-inventory/cloudbeat-aws-asset-inventory.yml similarity index 100% rename from deploy/aws-asset-inventory/cloudbeat-aws-asset-inventory.yml rename to deploy/asset-inventory/cloudbeat-aws-asset-inventory.yml diff --git a/deploy/asset-inventory/cloudbeat-gcp-asset-inventory.yml b/deploy/asset-inventory/cloudbeat-gcp-asset-inventory.yml new file mode 100644 index 0000000000..609f1c28ba --- /dev/null +++ b/deploy/asset-inventory/cloudbeat-gcp-asset-inventory.yml @@ -0,0 +1,66 @@ +cloudbeat: + type: cloudbeat/asset_inventory + config: + v1: + type: asset_inventory + asset_inventory_provider: gcp + gcp: + project_id: ${GCP_PROJECT_ID:""} + account_type: ${GCP_ACCOUNT_TYPE:""} + credentials: + credentials_json: ${GCP_CREDENTIALS_JSON:""} + # credentials_file_path: ${GOOGLE_APPLICATION_CREDENTIALS:""} + # Defines how often an event is sent to the output + period: 30s + evaluator: + decision_logs: false +# =================================== Kibana =================================== +setup.kibana: + # Kibana Host + host: "http://host.docker.internal:5601" +# =============================== Elastic Cloud ================================ + +# These settings simplify using Cloudbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +# ---------------------------- Elasticsearch Output ---------------------------- +output.elasticsearch: + # Array of hosts to connect to. + hosts: ${ES_HOST} + + # Protocol - either `http` (default) or `https`. + # protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + username: ${ES_USERNAME} + password: ${ES_PASSWORD} + + # Enable to allow sending output to older ES versions + allow_older_versions: true + # ssl.certificate_authorities: ${ES_CERT} + +# ================================= Processors ================================= +processors: + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - drop_fields: + fields: ["host.name"] +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +logging.level: debug +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publisher", "service" +# Multiple selectors can be chained. +#logging.selectors: ["publisher"] + +# Send all logging output to stderr. The default is false. +#logging.to_stderr: false diff --git a/tests/product/tests/data/gcp_asset_inventory/__init__.py b/tests/product/tests/data/gcp_asset_inventory/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/product/tests/data/gcp_asset_inventory/test_cases.py b/tests/product/tests/data/gcp_asset_inventory/test_cases.py new file mode 100644 index 0000000000..66c4a3564f --- /dev/null +++ b/tests/product/tests/data/gcp_asset_inventory/test_cases.py @@ -0,0 +1,92 @@ +""" +This module provides GCP test cases for Asset Inventory. +""" + +from ..asset_inventory_test_case import AssetInventoryCase + +test_cases = { + "[Asset Inventory][GCP][Service Account] assets found": AssetInventoryCase( + category="identity", + sub_category="service-identity", + type_="service-account", + sub_type="gcp-service-account", + ), + "[Asset Inventory][GCP][Service Account Key] assets found": AssetInventoryCase( + category="identity", + sub_category="service-identity", + type_="service-account-key", + sub_type="gcp-service-account-key", + ), + "[Asset Inventory][GCP][Instance] assets found": AssetInventoryCase( + category="infrastructure", + sub_category="compute", + type_="virtual-machine", + sub_type="gcp-instance", + ), + "[Asset Inventory][GCP][Subnet] assets found": AssetInventoryCase( + category="infrastructure", + sub_category="network", + type_="subnet", + sub_type="gcp-subnet", + ), + # "[Asset Inventory][GCP][Project] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="management", + # type_="cloud-account", + # sub_type="gcp-project", + # ), + # "[Asset Inventory][GCP][Organization] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="management", + # type_="cloud-account", + # sub_type="gcp-organization", + # ), + # "[Asset Inventory][GCP][Folder] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="management", + # type_="resource-hierarchy", + # sub_type="gcp-folder", + # ), + # "[Asset Inventory][GCP][Bucket] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="storage", + # type_="object-storage", + # sub_type="gcp-bucket", + # ), + # "[Asset Inventory][GCP][Firewall] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="network", + # type_="firewall", + # sub_type="gcp-firewall", + # ), + # "[Asset Inventory][GCP][GKE Cluster] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="container", + # type_="orchestration", + # sub_type="gcp-gke-cluster", + # ), + # "[Asset Inventory][GCP][Forwarding Rule] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="network", + # type_="load-balancing", + # sub_type="gcp-forwarding-rule", + # ), + # "[Asset Inventory][GCP][IAM Role] assets found": AssetInventoryCase( + # category="identity", + # sub_category="access-management", + # type_="iam-role", + # sub_type="gcp-iam-role", + # ), + # "[Asset Inventory][GCP][Cloud Function] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="serverless", + # type_="function", + # sub_type="gcp-cloud-function", + # ), + # "[Asset Inventory][GCP][Cloud Run Service] assets found": AssetInventoryCase( + # category="infrastructure", + # sub_category="container", + # type_="serverless", + # sub_type="gcp-cloud-run-service", + # ), +} diff --git a/tests/product/tests/test_gcp_asset_inventory.py b/tests/product/tests/test_gcp_asset_inventory.py new file mode 100644 index 0000000000..0706ff73cf --- /dev/null +++ b/tests/product/tests/test_gcp_asset_inventory.py @@ -0,0 +1,54 @@ +""" +GCP Asset Inventory Elastic Compute Cloud verification. +This module verifies presence and correctness of retrieved assets +""" + +from datetime import datetime, timedelta + +import pytest +from commonlib.utils import get_ES_assets +from product.tests.data.gcp_asset_inventory import test_cases as gcp_tc +from product.tests.parameters import Parameters, register_params + + +@pytest.mark.asset_inventory +@pytest.mark.asset_inventory_gcp +def test_gcp_asset_inventory( + asset_inventory_client, + category, + sub_category, + type_, + sub_type, +): + """ + This data driven test verifies assets published by cloudbeat agent. + """ + # pylint: disable=duplicate-code + assets = get_ES_assets( + asset_inventory_client, + timeout=10, + category=category, + sub_category=sub_category, + type_=type_, + sub_type=sub_type, + exec_timestamp=datetime.utcnow() - timedelta(minutes=30), + ) + + assert assets is not None, "Expected a list of assets, got None" + assert isinstance(assets, list) and len(assets) > 0, "Expected the list to be non-empty" + for asset in assets: + assert asset.cloud, "Expected .cloud section" + assert asset.cloud.provider == "gcp", f'Expected "gcp" provider, got {asset.cloud.provider}' + assert len(asset.asset.id) > 0, "Expected .asset.id list to contain an ID" + assert len(asset.asset.id[0]) > 0, "Expected the ID to be non-empty" + assert asset.asset.raw, "Expected the resource under .asset.raw" + + +register_params( + test_gcp_asset_inventory, + Parameters( + ("category", "sub_category", "type_", "sub_type"), + [*gcp_tc.test_cases.values()], + ids=[*gcp_tc.test_cases.keys()], + ), +) diff --git a/tests/pyproject.toml b/tests/pyproject.toml index 07c68c73ec..09a1d0b661 100644 --- a/tests/pyproject.toml +++ b/tests/pyproject.toml @@ -70,4 +70,5 @@ markers = [ "cspm_azure_microsoft_defender_rules", "asset_inventory", "asset_inventory_aws", + "asset_inventory_gcp", ]