Mark immutable config settings as such

A few of the yk8s config settings in `k8s-service-layer.prometheus` and
 `k8s-service-layer.rook` are actually immutable because they feed into
 the currently immutable `volumeClaimTemplate` field of Kubernetes objects.

Example: The `k8s-service-layer.prometheus.thanos_storegateway_size` yk8s
 config setting feeds into the `storegateway.persistence.size` parameter of the
 bitnami/thanos helm chart which feeds into the `volumeClaimTemplate` field of
 the `thanos-storegateway` statefulset created. That field is currently
 immutable making helm updates with the parameter fail.

Related-to: kubernetes/enhancements#4650

docs/_releasenotes/1498.fix.mark-immutable-config-settings-as-such

@@ -0,0 +1,11 @@
+Marked the following immutable config setting as such:
+- ``k8s-service-layer.prometheus.thanos_storegateway_size``
+- ``k8s-service-layer.prometheus.thanos_storegateway_size``
+- ``k8s-service-layer.prometheus.thanos_compactor_size``
+- ``k8s-service-layer.rook.mon_volume_storage_class``
+- ``k8s-service-layer.rook.mon_volume_size``
+- ``k8s-service-layer.rook.osd_volume_size``
+- ``k8s-service-layer.rook.osd_storage_class``
+
+A manual workaround to change these settings nonetheless
+is outlined in the respective documentation.

nix/yk8s/k8s-supplements/monitoring.nix

@@ -153,9 +153,16 @@ in {
         k8s-service-layer.prometheus.thanos_storegateway_size = "XGi";
         k8s-service-layer.prometheus.thanos_compactor_size = "YGi";
 
-      Options
-      *******
+      .. _cluster-configuration.prometheus-configuration.updating-immutable-options:
 
+      Updating immutable options
+      **************************
+
+      Some options are immutable when deployed.
+      If you want to change them nonetheless, follow these manual steps:
+      1. Increase the size of the corresponding PVC
+      2. Delete the stateful set: ``kubectl delete -n monitoring sts --cascade=false thanos-<storegateway|compactor>``
+      3. Re-deploy it with the LCM: ``AFLAGS="--diff --tags thanos" bash managed-k8s/actions/apply-k8s-supplements.sh``
     '';
 
     install = mkOption {
@@ -343,6 +350,8 @@ in {
       description = ''
         You can explicitly set the PV size for each component.
         If left undefined, the helm chart defaults will be used
+
+        Immutable when deployed. (See also :ref:`cluster-configuration.prometheus-configuration.updating-immutable-options`)
       '';
       type = with types; nullOr k8sSize;
       default = null;
@@ -352,6 +361,8 @@ in {
       description = ''
         You can explicitly set the PV size for each component.
         If left undefined, the helm chart defaults will be used
+
+        Immutable when deployed. (See also :ref:`cluster-configuration.prometheus-configuration.updating-immutable-options`)
       '';
       type = with types; nullOr k8sSize;
       default = null;

nix/yk8s/k8s-supplements/rook.nix

@@ -57,6 +57,17 @@ in {
         ``kubernetes.local_storage.dynamic.enabled`` or
         ``kubernetes.local_storage.static.enabled`` local
         storage (or both) (see :ref:`storage configuration <configuration-options.yk8s.kubernetes.storage>`).
+
+      .. _cluster-configuration.rook-configuration.updating-immutable-options:
+
+      Updating immutable options
+      **************************
+
+      Some options are immutable when deployed.
+      If you want to change them nonetheless, follow these manual steps:
+      1. Increase the size of the corresponding PVC
+      2. Delete the stateful set: ``kubectl delete -n monitoring sts --cascade=false <statefulset_name>``
+      3. Re-deploy it with the LCM: ``AFLAGS="--diff --tags rook --tags rook_v2" bash managed-k8s/actions/apply-k8s-supplements.sh``
     '';
 
     mon_allow_multiple_per_node = mkOption {
@@ -139,6 +150,9 @@ in {
     };
 
     mon_volume_size = mkOption {
+      description = ''
+        Immutable when deployed. (See also :ref:`cluster-configuration.rook-configuration.updating-immutable-options`)
+      '';
       type = k8sSize;
       default = "10Gi";
     };
@@ -149,6 +163,8 @@ in {
         storage class you have configured in the kubernetes.local_storage section (or
         you should know what your are doing). Note that this is not the storage class
         name that rook will provide.
+
+        Immutable when deployed. (See also :ref:`cluster-configuration.rook-configuration.updating-immutable-options`)
       '';
       type = types.nonEmptyStr;
       default = config.yk8s.kubernetes.local_storage.dynamic.storageclass_name;
@@ -264,12 +280,17 @@ in {
 
     osd_storage_class = mkOption {
       type = types.nonEmptyStr;
+      description = ''
+        Immutable when deployed. (See also :ref:`cluster-configuration.rook-configuration.updating-immutable-options`)
+      '';
       default = "csi-sc-cinderplugin";
     };
 
     osd_volume_size = mkOption {
       description = ''
         The size of the storage backing each OSD.
+
+        Immutable when deployed. (See also :ref:`cluster-configuration.rook-configuration.updating-immutable-options`)
       '';
       type = k8sSize;
       default = "90Gi";