[CIRCLECI] Dynamic-CI 💡

.circleci/scripts/generate_ci.py

@@ -0,0 +1,118 @@
+import os
+
+import yaml
+from jinja2 import Template
+
+# Define the top-level directories
+TOP_LEVEL_DIRS = [
+    "external-import",
+    "internal-enrichment",
+    "internal-export-file",
+    "internal-import-file",
+    "stream",
+]
+
+CI_DIR = ".circleci"
+TEMPLATE_DIR = f"{CI_DIR}/templates"
+TEMPLATE_PATH = f"{TEMPLATE_DIR}/dynamic.yml.j2"
+VARS_PATH = f"{CI_DIR}/vars.yml"
+
+REPOSITORY = "opencti"
+
+
+def get_dirs() -> dir:
+    # Collect subdirectories for each top-level directory
+    dirs = {}
+    for top_dir in TOP_LEVEL_DIRS:
+        if os.path.exists(top_dir):
+            dirs[top_dir] = [
+                sub_dir
+                for sub_dir in os.listdir(top_dir)
+                if os.path.isdir(os.path.join(top_dir, sub_dir))
+            ]
+    return dirs
+
+
+# Load the Jinja template
+def get_parameters() -> dict:
+    """Load the parameters contained in a yaml file
+
+    The loaded dictionary describes image-specific details that deviate from the default case.
+    {
+        "images":
+        {
+            <name_of_the_image>:
+            {
+                "python" : specific python version
+                "fips": boolean to tell the pipeline to build an image with "fips" tag
+            },
+            ...
+        }
+    }
+    """
+
+    with open(f"{CI_DIR}/vars.yml", "r") as yaml_file:
+        return yaml.safe_load(yaml_file)
+
+
+def get_template() -> Template:
+    with open(TEMPLATE_PATH, "r") as template_file:
+        return Template(template_file.read())
+
+
+def get_pycti() -> dict:
+    """
+    Retrieve the pycti configuration, including its version and whether it should be replaced.
+
+    This function checks the environment for a `CIRCLE_TAG` variable to determine the pycti version.
+    `CIRCLE_TAG` only exists when release is done
+
+    If 'replace' flag is set to true, we will pull the pycti dependency from a specific branch rather than from the pypi registry
+
+    :return: pycti version
+    """
+    pycti = {"version": os.getenv("CIRCLE_TAG")}
+    if pycti["version"]:
+        pycti["replace"] = False
+    else:
+        pycti["replace"] = True
+    return pycti
+
+
+def get_tags() -> list[str]:
+    data = []
+    tags = os.getenv("BUILD_TAGS")
+    if tags:
+        data.extend(tags.split(","))
+
+    circle_tag = os.getenv("CIRCLE_TAG")
+    if circle_tag:
+        data.append(circle_tag)
+    if len(data) == 0:
+        print("[ERROR]: At least 1 tag is required")
+        exit(1)
+    return data
+
+
+# Write the generated config to a CircleCI configuration file
+def write_config(template):
+    output_path = "dynamic.yml"
+    with open(output_path, "w") as file:
+        file.write(template)
+    print(f"Generated CircleCI config at {output_path}")
+
+
+def main():
+    template = get_template()
+    config = template.render(
+        dirs=get_dirs(),
+        param=get_parameters(),
+        pycti=get_pycti(),
+        tags=get_tags(),
+        repo=REPOSITORY,
+    )
+    write_config(config)
+
+
+if __name__ == "__main__":
+    main()

.circleci/scripts/requirements.txt

@@ -0,0 +1,3 @@
+Jinja2==3.1.2
+PyYAML==6.0.1
+requests==2.32.3

.circleci/templates/dynamic.yml.j2

@@ -0,0 +1,66 @@
+version: 2.1
+
+jobs:
+{% for top_dir, sub_dirs in dirs.items() -%}
+{% for sub_dir in sub_dirs %}
+  build_{{top_dir}}_{{sub_dir}}:
+    docker:
+      - image: cimg/base:stable-20.04
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Build {{top_dir}}/{{sub_dir}}
+          command: |
+            echo "Processing {{top_dir}}/{{sub_dir}}"
+            cd {{top_dir}}/{{sub_dir}}
+            docker run --privileged --rm tonistiigi/binfmt --install linux/amd64,linux/arm64
+            CIRCLE_TAG=${CIRCLE_TAG:-latest}
+            docker buildx create --use --name builder-connector-{{sub_dir}}
+            docker buildx inspect builder-connector-{{sub_dir}} --bootstrap
+            echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+            echo "$GHCR_PASS" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+            {% set image_key = top_dir ~ '_' ~ sub_dir -%}
+
+            {% if param['images'][image_key] is defined -%}
+            base_image="{{repo}}/client-python-{{param['images'][image_key].python}}:{{version}}"
+            {% else -%}
+            base_image="{{repo}}/client-python-3-12:{{version}}"
+            {% endif -%}
+
+            {% if pycti.replace -%}
+                {% if "prerelease" in tags %}
+            find . -name requirements.txt -exec sed 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/client-python@$CIRCLE_BRANCH|' -i {} \;
+                {% elif "rolling" in tags %}
+            find . -name requirements.txt -exec sed 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/client-python@master|' -i {} \;
+                {% endif %}
+            {% endif -%}
+            docker buildx build . \
+             {% for tag in tags -%}
+             --tag {{repo}}/connector-{{sub_dir}}:{{tag}} \
+             --tag ghcr.io/opencti-platform/{{repo}}/connector-{{sub_dir}}:{{tag}} \
+             {% endfor -%}
+             --build-arg BASE_IMAGE=$base_image \
+             --push
+
+            {% if param['images'][image_key] is defined and param['images'][image_key]['fips'] == true -%}
+            docker buildx build -f Dockerfile_fips . \
+             {% for tag in tags -%}
+             --tag {{repo}}/connector-{{sub_dir}}:{{tag}}-fips \
+             --tag ghcr.io/opencti-platform/{{repo}}/connector-{{sub_dir}}:{{tag}}-fips \
+             {% endfor -%}
+             --build-arg BASE_IMAGE=$base_image \
+             --push
+            {% endif -%}
+
+      {% endfor %}
+{% endfor %}
+workflows:
+  version: 2.1
+  build_connectors:
+    jobs:
+  {%- for top_dir, sub_dirs in dirs.items() -%}
+  {%- for sub_dir in sub_dirs %}
+      - build_{{top_dir}}_{{sub_dir}}
+  {%- endfor -%}
+{% endfor-%}

.circleci/vars.yml

@@ -0,0 +1,29 @@
+# List of images with a different configuration than the default one 
+# default is  a base python image 3.12,  built with tag "rolling" or release semantic version
+images:
+  stream_webhook:
+    python: python3-11
+  stream_qradar:
+    python: python3-11
+  stream_splunk:
+    python: python3-11
+  stream_sentinel-intel:
+    python: python3-11
+  stream_sentinel:
+    python: python3-11
+  stream_harfanglab-intel:
+    python: python3-11
+  stream_harfanglab:
+    python: python3-11
+  external-import_harfanglab-incidents:
+    python: python3-11
+  internal-import-file_import-file-stix:
+    fips: true
+  internal-import-file_import-document:
+    fips: true
+  internal-export-file_export-file-stix:
+    fips: true
+  internal-export-file_export-file-csv:
+    fips: true
+  internal-export-file_export-file-txt:
+    fips: true

external-import/ironnet/Dockerfile

@@ -6,12 +6,14 @@ COPY src /opt/opencti-connector-ironnet
 
 # Install Python modules
 # hadolint ignore=DL3003
-RUN apk --no-cache add git build-base libmagic libffi-dev && \
-    cd /opt/opencti-connector-ironnet && \
+RUN apk update && apk upgrade && \
+    apk --no-cache add git build-base libmagic libffi-dev libxml2-dev libxslt-dev
+
+RUN cd /opt/opencti-connector-ironnet && \
     pip3 install --no-cache-dir -r requirements.txt && \
     apk del git build-base
 
 # Expose and entrypoint
 COPY entrypoint.sh /
 RUN chmod +x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]

external-import/sentinelone-threats/Dockerfile

@@ -6,8 +6,10 @@ COPY src /opt/opencti-connector-sentinelone-threats
 
 # Install Python modules
 # hadolint ignore=DL3003
-RUN apk --no-cache add git build-base libmagic libffi-dev && \
-    cd /opt/opencti-connector-sentinelone-threats && \
+RUN apk update && apk upgrade && \
+    apk --no-cache add git build-base libmagic libffi-dev libxml2-dev libxslt-dev
+
+RUN cd /opt/opencti-connector-sentinelone-threats && \
     pip3 install --no-cache-dir -r requirements.txt && \
     apk del git build-base