[Assets Inventory] Add GCP integration tests to CI workflow (#2499)

add gcp asset inventory integration tests and ci workflow

.github/actions/aws-asset-inventory-ci/action.yml

@@ -37,7 +37,7 @@ runs:
         AWS_ACCOUNT_TYPE: ${{ inputs.aws-account-type }}
       shell: bash
       run: |
-        ./cloudbeat -c deploy/aws-asset-inventory/cloudbeat-aws-asset-inventory.yml -d '*' &
+        ./cloudbeat -c deploy/asset-inventory/cloudbeat-aws-asset-inventory.yml -d '*' &
 
     - name: Wait for cloudbeat to send some events
       shell: bash

.github/actions/gcp-asset-inventory-ci/action.yml

@@ -0,0 +1,65 @@
+name: "GCP Asset Inventory CI"
+description: "GCP Asset Inventory integration tests"
+inputs:
+  elk-version:
+    description: "ELK version"
+    required: true
+  credentials-json:
+    description: "GCP Service account key JSON"
+    required: true
+  project-id:
+    description: "GCP Project ID"
+    required: true
+
+  debug:
+    description: "debug"
+    required: false
+    default: "false"
+runs:
+  using: composite
+  steps:
+    - name: Init Integration
+      uses: ./.github/actions/init-integration
+      with:
+        elk-version: ${{ inputs.elk-version }}
+
+    - name: Run cloudbeat in background
+      env:
+        ES_HOST: http://localhost:9200
+        ES_USERNAME: elastic
+        ES_PASSWORD: changeme
+        GCP_PROJECT_ID: ${{ inputs.project-id}}
+        GCP_CREDENTIALS_JSON: ${{ inputs.credentials-json }}
+        GCP_ACCOUNT_TYPE: single-account
+      shell: bash
+      run: |
+        ./cloudbeat -c deploy/asset-inventory/cloudbeat-gcp-asset-inventory.yml -d '*' &
+
+    - name: Wait for cloudbeat to send some events
+      shell: bash
+      run: sleep 20
+
+    - name: Check for assets
+      working-directory: ./tests
+      env:
+        USE_K8S: "false"
+      shell: bash
+      run: poetry run pytest -k "asset_inventory_gcp" --alluredir=./allure/results/ --clean-alluredir
+
+    - name: Upload test results
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: allure-results-ci-gcp-asset-inventory
+        path: tests/allure/results/
+        overwrite: true
+
+    - if: ${{ failure() || cancelled() || inputs.debug == 'true' }}
+      name: Upload cloudbeat logs
+      uses: actions/upload-artifact@v4
+      with:
+        name: cloubeat-logs-ci-gcp-asset-inventory
+        path: logs/
+        if-no-files-found: warn
+        retention-days: 1
+        overwrite: true

.github/workflows/ci.yml

@@ -98,6 +98,28 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_TEST_ACC }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_TEST_ACC }}
 
+  ci-gcp-asset-inventory:
+    needs: [init-hermit]
+    name: GCP Asset Inventory CI
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Hermit Environment
+        uses: ./.github/actions/hermit
+
+      - name: Run GCP Asset Inventory integration tests
+        uses: ./.github/actions/gcp-asset-inventory-ci
+        with:
+          elk-version: ${{ env.ELK_VERSION }}
+          credentials-json: ${{ secrets.GCP_ASSETS_INVENTORY_CREDENTIALS_JSON }}
+          project-id: "elastic-security-test"
+
   ci-gcp:
     needs: [ init-hermit ]
     name: CIS GCP CI

deploy/asset-inventory/cloudbeat-gcp-asset-inventory.yml

@@ -0,0 +1,66 @@
+cloudbeat:
+  type: cloudbeat/asset_inventory
+  config:
+    v1:
+      type: asset_inventory
+      asset_inventory_provider: gcp
+      gcp:
+        project_id: ${GCP_PROJECT_ID:""}
+        account_type: ${GCP_ACCOUNT_TYPE:""}
+        credentials:
+          credentials_json: ${GCP_CREDENTIALS_JSON:""}
+          # credentials_file_path: ${GOOGLE_APPLICATION_CREDENTIALS:""}
+  # Defines how often an event is sent to the output
+  period: 30s
+  evaluator:
+    decision_logs: false
+# =================================== Kibana ===================================
+setup.kibana:
+  # Kibana Host
+  host: "http://host.docker.internal:5601"
+# =============================== Elastic Cloud ================================
+
+# These settings simplify using Cloudbeat with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
+# ---------------------------- Elasticsearch Output ----------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ${ES_HOST}
+
+  # Protocol - either `http` (default) or `https`.
+  #  protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
+  username: ${ES_USERNAME}
+  password: ${ES_PASSWORD}
+
+  # Enable to allow sending output to older ES versions
+  allow_older_versions: true
+  # ssl.certificate_authorities: ${ES_CERT}
+
+# ================================= Processors =================================
+processors:
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - drop_fields:
+      fields: ["host.name"]
+# Sets log level. The default log level is info.
+# Available log levels are: error, warning, info, debug
+logging.level: debug
+# Enable debug output for selected components. To enable all selectors use ["*"]
+# Other available selectors are "beat", "publisher", "service"
+# Multiple selectors can be chained.
+#logging.selectors: ["publisher"]
+
+# Send all logging output to stderr. The default is false.
+#logging.to_stderr: false

tests/product/tests/data/gcp_asset_inventory/test_cases.py

@@ -0,0 +1,92 @@
+"""
+This module provides GCP test cases for Asset Inventory.
+"""
+
+from ..asset_inventory_test_case import AssetInventoryCase
+
+test_cases = {
+    "[Asset Inventory][GCP][Service Account] assets found": AssetInventoryCase(
+        category="identity",
+        sub_category="service-identity",
+        type_="service-account",
+        sub_type="gcp-service-account",
+    ),
+    "[Asset Inventory][GCP][Service Account Key] assets found": AssetInventoryCase(
+        category="identity",
+        sub_category="service-identity",
+        type_="service-account-key",
+        sub_type="gcp-service-account-key",
+    ),
+    "[Asset Inventory][GCP][Instance] assets found": AssetInventoryCase(
+        category="infrastructure",
+        sub_category="compute",
+        type_="virtual-machine",
+        sub_type="gcp-instance",
+    ),
+    "[Asset Inventory][GCP][Subnet] assets found": AssetInventoryCase(
+        category="infrastructure",
+        sub_category="network",
+        type_="subnet",
+        sub_type="gcp-subnet",
+    ),
+    # "[Asset Inventory][GCP][Project] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="management",
+    #     type_="cloud-account",
+    #     sub_type="gcp-project",
+    # ),
+    # "[Asset Inventory][GCP][Organization] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="management",
+    #     type_="cloud-account",
+    #     sub_type="gcp-organization",
+    # ),
+    # "[Asset Inventory][GCP][Folder] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="management",
+    #     type_="resource-hierarchy",
+    #     sub_type="gcp-folder",
+    # ),
+    # "[Asset Inventory][GCP][Bucket] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="storage",
+    #     type_="object-storage",
+    #     sub_type="gcp-bucket",
+    # ),
+    # "[Asset Inventory][GCP][Firewall] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="network",
+    #     type_="firewall",
+    #     sub_type="gcp-firewall",
+    # ),
+    # "[Asset Inventory][GCP][GKE Cluster] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="container",
+    #     type_="orchestration",
+    #     sub_type="gcp-gke-cluster",
+    # ),
+    # "[Asset Inventory][GCP][Forwarding Rule] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="network",
+    #     type_="load-balancing",
+    #     sub_type="gcp-forwarding-rule",
+    # ),
+    # "[Asset Inventory][GCP][IAM Role] assets found": AssetInventoryCase(
+    #     category="identity",
+    #     sub_category="access-management",
+    #     type_="iam-role",
+    #     sub_type="gcp-iam-role",
+    # ),
+    # "[Asset Inventory][GCP][Cloud Function] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="serverless",
+    #     type_="function",
+    #     sub_type="gcp-cloud-function",
+    # ),
+    # "[Asset Inventory][GCP][Cloud Run Service] assets found": AssetInventoryCase(
+    #     category="infrastructure",
+    #     sub_category="container",
+    #     type_="serverless",
+    #     sub_type="gcp-cloud-run-service",
+    # ),
+}

tests/product/tests/test_gcp_asset_inventory.py

@@ -0,0 +1,54 @@
+"""
+GCP Asset Inventory Elastic Compute Cloud verification.
+This module verifies presence and correctness of retrieved assets
+"""
+
+from datetime import datetime, timedelta
+
+import pytest
+from commonlib.utils import get_ES_assets
+from product.tests.data.gcp_asset_inventory import test_cases as gcp_tc
+from product.tests.parameters import Parameters, register_params
+
+
+@pytest.mark.asset_inventory
+@pytest.mark.asset_inventory_gcp
+def test_gcp_asset_inventory(
+    asset_inventory_client,
+    category,
+    sub_category,
+    type_,
+    sub_type,
+):
+    """
+    This data driven test verifies assets published by cloudbeat agent.
+    """
+    # pylint: disable=duplicate-code
+    assets = get_ES_assets(
+        asset_inventory_client,
+        timeout=10,
+        category=category,
+        sub_category=sub_category,
+        type_=type_,
+        sub_type=sub_type,
+        exec_timestamp=datetime.utcnow() - timedelta(minutes=30),
+    )
+
+    assert assets is not None, "Expected a list of assets, got None"
+    assert isinstance(assets, list) and len(assets) > 0, "Expected the list to be non-empty"
+    for asset in assets:
+        assert asset.cloud, "Expected .cloud section"
+        assert asset.cloud.provider == "gcp", f'Expected "gcp" provider, got {asset.cloud.provider}'
+        assert len(asset.asset.id) > 0, "Expected .asset.id list to contain an ID"
+        assert len(asset.asset.id[0]) > 0, "Expected the ID to be non-empty"
+        assert asset.asset.raw, "Expected the resource under .asset.raw"
+
+
+register_params(
+    test_gcp_asset_inventory,
+    Parameters(
+        ("category", "sub_category", "type_", "sub_type"),
+        [*gcp_tc.test_cases.values()],
+        ids=[*gcp_tc.test_cases.keys()],
+    ),
+)

tests/pyproject.toml

@@ -70,4 +70,5 @@ markers = [
     "cspm_azure_microsoft_defender_rules",
     "asset_inventory",
     "asset_inventory_aws",
+    "asset_inventory_gcp",
 ]