Allow to disable loki and fluent-bit (and to configure fluent-bit output)

.github/actions/generate-bootstrap/action.yaml

@@ -13,6 +13,10 @@ inputs:
     description: "Whether or not Dex should be enabled"
     required: false
     default: "true"
+  LOGGING_ENABLED:
+    description: "Wether or not logging should be enabled"
+    required: false
+    default: "true"
   DEBUG:
     description: "Whether or not debug mode should be enabled"
     required: false
@@ -60,6 +64,10 @@ runs:
           addons:
             dex:
               enabled: ${{ inputs.DEX_ENABLED }}
+            loki:
+              enabled: ${{ inputs.LOGGING_ENABLED }}
+            fluent-bit:
+              enabled: ${{ inputs.LOGGING_ENABLED }}
           kubernetes:
             coreDNS:
               affinity: ${{ env.AFFINITY }}

.github/actions/provision-volumes/action.yaml

@@ -10,6 +10,10 @@ inputs:
     description: "Path to the MetalK8s ISO mountpoint on bootstrap node"
     required: false
     default: "/var/tmp/metalk8s"
+  LOGGING_ENABLED:
+    description: "Wether or not logging should be enabled"
+    required: false
+    default: "true"
 
 runs:
   using: "composite"
@@ -27,4 +31,5 @@ runs:
           PRODUCT_MOUNT=\"${{ inputs.MOUNTPOINT }}\"
           PRODUCT_TXT=\"${{ inputs.MOUNTPOINT }}/product.txt\"
           NODE_NAME=\"${{ inputs.NODE }}\"
+          LOKI_ENABLED=\"${{ inputs.LOGGING_ENABLED }}\"
           /tmp/create-volumes.sh

.github/scripts/create-volumes.sh

@@ -39,6 +39,7 @@ check_pod_is_in_phase() {
 
 NODE_NAME=${NODE_NAME:-$(salt-call --local --out txt grains.get id | cut -c 8-)}
 PRODUCT_TXT=${PRODUCT_TXT:-/vagrant/_build/root/product.txt}
+LOKI_ENABLED=${LOKI_ENABLED:-true}
 MAX_TRIES=300
 
 # shellcheck disable=SC1090
@@ -93,5 +94,7 @@ wait_for_pod "AlertManager" \
     metalk8s-monitoring alertmanager-prometheus-operator-alertmanager-0
 wait_for_pod "Prometheus" \
     metalk8s-monitoring prometheus-prometheus-operator-prometheus-0
-wait_for_pod "Loki" \
-    metalk8s-logging loki-0
+if [ "$LOKI_ENABLED" = "true" ]; then
+    wait_for_pod "Loki" \
+        metalk8s-logging loki-0
+fi

.github/workflows/nightly.yaml

@@ -171,14 +171,17 @@ jobs:
           # we also test solution install + upgrade there
           - rocky-8
         setup-idp: [true]
+        setup-logging: [true]
         include:
           - os: rocky-8
             setup-idp: false
+            setup-logging: false
     with:
       artifacts-url: ${{ needs.retrieve-info.outputs.artifacts-link }}
       os: ${{ matrix.os }}
       test-solution: true
       setup-idp: ${{ matrix.setup-idp }}
+      setup-logging: ${{ matrix.setup-logging }}
 
   bootstrap-restore:
     uses: ./.github/workflows/multi-node-test.yaml

.github/workflows/single-node-test.yaml

@@ -43,6 +43,11 @@ on:
         required: false
         type: boolean
         default: true
+      setup-logging:
+        description: "Whether or not the logging should be configured and tested"
+        required: false
+        type: boolean
+        default: true
 
       # Solution information
       solution-name:
@@ -72,7 +77,7 @@ on:
         default: "1.0.3"
 
 env:
-  NAME: single-node-${{ inputs.os }}-${{ inputs.setup-idp && 'idp' || 'no-idp' }}
+  NAME: single-node-${{ inputs.os }}-${{ inputs.setup-idp && 'idp' || 'no-idp' }}-${{ inputs.setup-logging && 'logging' || 'no-logging' }}
 
 jobs:
   single-node:
@@ -144,6 +149,7 @@ jobs:
         uses: ./.github/actions/generate-bootstrap
         with:
           DEX_ENABLED: ${{ inputs.setup-idp }}
+          LOGGING_ENABLED: ${{ inputs.setup-logging }}
       - name: Mount MetalK8s ISO
         uses: ./.github/actions/mount-iso
       - name: Run Bootstrap
@@ -154,6 +160,8 @@ jobs:
           metalk8s-version: ${{ env.METALK8S_VERSION }}
       - name: Provision volumes
         uses: ./.github/actions/provision-volumes
+        with:
+          LOGGING_ENABLED: ${{ inputs.setup-logging }}
       - name: Untaint bootstrap node
         uses: ./.github/actions/untaint
       # }}}
@@ -224,7 +232,9 @@ jobs:
       - name: Run tests from Bastion
         uses: ./.github/actions/bastion-tests
         with:
-          EXTRA_FILTER: ${{ inputs.setup-idp && ' ' || 'and not authentication' }}
+          EXTRA_FILTER: >-
+            ${{ inputs.setup-idp && ' ' || 'and not authentication' }}
+            ${{ inputs.setup-logging && ' ' || 'and not logging' }}
       - name: Run UI tests from Bastion
         if: inputs.setup-idp
         uses: ./.github/actions/bastion-ui-tests

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Release 127.0.0 (in development)
 
+### Additions
+
+- Add ability to configure fluent-bit output
+  (PR[#4276](https://github.com/scality/metalk8s/pull/4276))
+
+- Add ability to deploy without Loki and Fluent-bit
+  (PR[#4276](https://github.com/scality/metalk8s/pull/4276))
+
 ### Enhancements
 
 - Bump Kubernetes version to

buildchain/buildchain/codegen.py

@@ -107,7 +107,7 @@ def codegen_chart_fluent_bit() -> types.TaskDict:
         f"{constants.CHART_RENDER_CMD} fluent-bit {value_file} {chart_dir} "
         "--namespace metalk8s-logging "
         "--service-config fluent_bit metalk8s-fluent-bit-config "
-        "metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml metalk8s-logging "
+        "metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml.j2 metalk8s-logging "
         f"--output {target_sls}"
     )
 

buildchain/buildchain/salt_tree.py

@@ -316,7 +316,7 @@ def task(self) -> types.TaskDict:
     Path("salt/metalk8s/addons/logging/deployed/files/fluent-bit.json"),
     Path("salt/metalk8s/addons/logging/deployed/files/logs.json"),
     Path("salt/metalk8s/addons/logging/deployed/files/loki.json"),
-    Path("salt/metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml"),
+    Path("salt/metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml.j2"),
     Path("salt/metalk8s/addons/logging/fluent-bit/deployed/chart.sls"),
     Path("salt/metalk8s/addons/logging/fluent-bit/deployed/configmap.sls"),
     Path("salt/metalk8s/addons/logging/fluent-bit/deployed/init.sls"),

docs/installation/bootstrap.rst

@@ -71,6 +71,10 @@ Configuration
       addons:
         dex:
           enabled: True
+        fluent-bit:
+          enabled: True
+        loki:
+          enabled: True
       kubernetes:
         apiServer:
           oidc: {}
@@ -189,6 +193,10 @@ to configure.
   If ``dex`` is disabled you will not be able to use the MetalK8s UI and
   Grafana.
 
+  Deployment of logging stack relying on fluent-bit and loki can be disabled
+  respectively by setting ``addons.fluent-bit.enabled`` and
+  ``addons.loki.enabled`` to ``false``.
+
 The ``kubernetes`` field can be omitted if you do not have any specific
 Kubernetes `Feature Gates`_ to enable or disable and if you are ok with
 defaults kubernetes configuration.

docs/operation/cluster_and_service_configuration.rst

@@ -95,9 +95,9 @@ forward them to a log aggregation system (loki).
 
 The default configuration values for Fluent-bit are specified below:
 
-.. literalinclude:: ../../salt/metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml
+.. literalinclude:: ../../salt/metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml.j2
    :language: yaml
-   :lines: 3-
+   :lines: 3-14,16-37
 
 
 UI Default Configuration

salt/_modules/metalk8s_kubernetes.py

@@ -130,6 +130,20 @@ def method(
                 f"Must provide one of {' or '.join(needed_params)} to {action} object."
             )
 
+        if action in ["delete", "get"]:
+            # Only keep needed information and remove anything else
+            # to avoid invalid error on slots formatting
+            manifest = {
+                "apiVersion": manifest["apiVersion"],
+                "kind": manifest["kind"],
+                "metadata": {
+                    "name": manifest.get("metadata", {}).get("name"),
+                    "namespace": manifest.get("metadata", {}).get(
+                        "namespace", namespace
+                    ),
+                },
+            }
+
         # Format slots on the manifest
         manifest = __salt__.metalk8s.format_slots(manifest)
 

salt/_pillar/metalk8s.py

@@ -166,6 +166,8 @@ def _load_addons(config_data):
     addons_data = config_data.get("addons", {})
 
     addons_data.setdefault("dex", {}).setdefault("enabled", True)
+    addons_data.setdefault("loki", {}).setdefault("enabled", True)
+    addons_data.setdefault("fluent-bit", {}).setdefault("enabled", True)
 
     return addons_data
 

salt/_renderers/metalk8s_kubernetes.py

@@ -75,6 +75,13 @@ def render(
     context = args.get("context", [None])[0]
     absent = args.get("absent", [False])[0]
 
+    # Allow to force absent arg from pillar
+    if (
+        __pillar__.get("_metalk8s_kubernetes_renderer", {}).get("force_absent")
+        is not None
+    ):
+        absent = __pillar__["_metalk8s_kubernetes_renderer"]["force_absent"]
+
     if not isinstance(source, six.string_types):
         # Assume it is a file handle
         source = source.read()

salt/metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml.j2

@@ -0,0 +1,40 @@
+#!jinja|yaml
+
+# Configuration of the fluent-bit service
+apiVersion: addons.metalk8s.scality.com
+kind: FluentBitConfig
+spec:
+  deployment:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 200Mi
+      limits:
+        memory: 1Gi
+  config:
+    {%- if pillar.addons.loki.enabled %}
+    output:
+      - Name: loki
+        Match: kube.*
+        Host: loki
+        Port: 3100
+        Tenant_ID: '""'
+        Labels: job=fluent-bit
+        Label_Keys: "$container, $node, $namespace, $pod, $app, $release, $stream"
+        Auto_Kubernetes_Labels: false
+        Line_Format: json
+        Log_Level: warn
+        Workers: 4
+      - Name: loki
+        Match: host.*
+        Host: loki
+        Port: 3100
+        Tenant_ID: '""'
+        Labels: "job=fluent-bit"
+        Label_Keys: "$hostname, $unit"
+        Line_Format: json
+        Log_Level: warn
+        Workers: 4
+    {%- else %}
+    output: []
+    {%- endif %}

salt/metalk8s/addons/logging/fluent-bit/deployed/chart.sls

@@ -2,7 +2,7 @@
 
 {%- from "metalk8s/map.jinja" import repo with context %}
 {%- from "metalk8s/repo/macro.sls" import build_image_name with context %}
-{%- set fluent_bit_defaults = salt.slsutil.renderer('salt://metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml', saltenv=saltenv) %}
+{%- set fluent_bit_defaults = salt.slsutil.renderer('salt://metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml.j2', saltenv=saltenv) %}
 {%- set fluent_bit = salt.metalk8s_service_configuration.get_service_conf('metalk8s-logging', 'metalk8s-fluent-bit-config', fluent_bit_defaults) %}
 
 {% raw %}

salt/metalk8s/addons/logging/fluent-bit/deployed/configmap.sls

@@ -1,6 +1,15 @@
 include:
   - ...deployed.namespace
 
+{%- set fluent_bit_defaults = salt.slsutil.renderer(
+        'salt://metalk8s/addons/logging/fluent-bit/config/fluent-bit.yaml.j2', saltenv=saltenv
+    )
+%}
+{%- set fluent_bit = salt.metalk8s_service_configuration.get_service_conf(
+        'metalk8s-logging', 'metalk8s-fluent-bit-config', fluent_bit_defaults
+    )
+%}
+
 Create fluent-bit ConfigMap:
   metalk8s_kubernetes.object_present:
     - manifest:
@@ -108,29 +117,12 @@ Create fluent-bit ConfigMap:
                 Add            namespace unknown
                 Add            pod unknown
                 Add            stream unknown
+            {%- for output in fluent_bit.spec.config.output %}
             [Output]
-                Name           loki
-                Match          kube.*
-                Host           loki
-                Port           3100
-                Tenant_ID      ""
-                Labels         job=fluent-bit
-                Label_Keys     $container, $node, $namespace, $pod, $app, $release, $stream
-                Auto_Kubernetes_Labels false
-                Line_Format    json
-                Log_Level      warn
-                Workers        4
-            [Output]
-                Name           loki
-                Match          host.*
-                Host           loki
-                Port           3100
-                Tenant_ID      ""
-                Labels         job=fluent-bit
-                Label_Keys     $hostname, $unit
-                Line_Format    json
-                Log_Level      warn
-                Workers        4
+                {%- for key, value in output.items() %}
+                {{ "{:<14} {}".format(key, value) }}
+                {%- endfor %}
+            {%- endfor %}
           custom_parsers.conf: |-
             [PARSER]
                 Name        container

salt/metalk8s/addons/logging/fluent-bit/deployed/init.sls

@@ -1,4 +1,32 @@
+{%- if pillar.addons["fluent-bit"].enabled %}
+
 include:
   - .configmap
   - .service-configuration
   - .chart
+
+{%- else %}
+
+{#- NOTE: We do not delete the CSC configuration as we do not
+    want to lose the configuration if the user re-enables the
+    addon. #}
+
+Ensure fluent-bit objects does not exist:
+  salt.runner:
+    - name: state.orchestrate
+    - pillar:
+        _metalk8s_kubernetes_renderer:
+          force_absent: True
+    - mods:
+      - metalk8s.addons.logging.fluent-bit.deployed.chart
+
+Ensure fluent-bit config does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: fluent-bit
+    - namespace: metalk8s-logging
+    - kind: ConfigMap
+    - apiVersion: v1
+    - require:
+      - salt: Ensure fluent-bit objects does not exist
+
+{%- endif %}