Use object instead of array in config.yml for config template

airflow/config_templates/config.yml.schema.json

@@ -1,61 +1,87 @@
 {
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "type": "array",
-  "items": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
-    "properties": {
-      "name": {
-        "type": "string"
-      },
-      "description": {
-        "type": ["string", "null"]
-      },
-      "options": {
-        "type": "array",
-        "items": {
-          "type": "object",
-          "properties": {
-            "name": {
-              "type": "string"
-            },
+    "additionalProperties": {
+        "type": "object",
+        "properties": {
             "description": {
-              "type": ["string", "null"]
-            },
-            "version_added": {
-              "type": ["string", "null"]
-            },
-            "type": {
-              "type": "string",
-              "enum": ["string", "boolean", "integer", "float"]
-            },
-            "example": {
-              "type": ["string", "null", "number"]
+                "type": [
+                    "string",
+                    "null"
+                ]
             },
-            "default": {
-              "type": ["string", "null", "number"]
+            "options": {
+                "type": "object",
+                "additionalProperties": {
+                    "$ref": "#/definitions/option"
+                }
             },
-            "sensitive": {
-              "type": "boolean",
-              "description": "When true, this option is sensitive and can be specified using AIRFLOW__{section}___{name}__SECRET or AIRFLOW__{section}___{name}_CMD environment variables. See: airflow.configuration.AirflowConfigParser.sensitive_config_values"
+            "renamed": {
+                "type": "object",
+                "properties": {
+                    "previous_name": {"type": "string"},
+                    "version": {"type": "string"}
+                }
             }
-          },
-          "required": [
-            "name",
+        },
+        "required": [
             "description",
-            "version_added",
-            "type",
-            "example",
-            "default"
-          ],
-          "additional_properties": false
-        }
-      }
+            "options"
+        ],
+        "additionalProperties": false
     },
-    "required": [
-      "name",
-      "description",
-      "options"
-    ],
-    "additional_properties": false
-  }
+    "definitions": {
+        "option": {
+            "type": "object",
+            "properties": {
+                "description": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                },
+                "version_added": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
+                },
+                "type": {
+                    "type": "string",
+                    "enum": [
+                        "string",
+                        "boolean",
+                        "integer",
+                        "float"
+                    ]
+                },
+                "example": {
+                    "type": [
+                        "string",
+                        "null",
+                        "number"
+                    ]
+                },
+                "default": {
+                    "type": [
+                        "string",
+                        "null",
+                        "number"
+                    ]
+                },
+                "sensitive": {
+                    "type": "boolean",
+                    "description": "When true, this option is sensitive and can be specified using AIRFLOW__{section}___{name}__SECRET or AIRFLOW__{section}___{name}_CMD environment variables. See: airflow.configuration.AirflowConfigParser.sensitive_config_values"
+                }
+            },
+            "required": [
+                "description",
+                "version_added",
+                "type",
+                "example",
+                "default"
+            ],
+            "additional_properties": false
+        }
+    }
 }

airflow/configuration.py

@@ -136,7 +136,7 @@ def _default_config_file_path(file_name: str) -> str:
     return os.path.join(templates_dir, file_name)
 
 
-def default_config_yaml() -> list[dict[str, Any]]:
+def default_config_yaml() -> dict[str, Any]:
     """
     Read Airflow configs from YAML file.
 

airflow/providers/google/config_templates/config.yml

@@ -15,10 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-- name: providers_google
+providers_google:
   description: Options for google provider
   options:
-    - name: verbose_logging
+    verbose_logging:
       description: |
         Sets verbose logging for google provider
       version_added: 2.0.0

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -325,7 +325,7 @@ def build_docs(
 ):
     """Build documentation in the container."""
     if for_production and not clean_build:
-        get_console().print("\n[warning]When building docs for production, clan-build is forced\n")
+        get_console().print("\n[warning]When building docs for production, clean-build is forced\n")
         clean_build = True
     perform_environment_checks()
     cleanup_python_generated_files()

docs/apache-airflow/configurations-ref.rst

@@ -41,12 +41,12 @@ that you run airflow components on is synchronized (for example using ntpd) othe
 
 .. jinja:: config_ctx
 
-    {% for section in configs %}
+    {% for section_name, section in configs.items() %}
 
-    .. _config:{{ section["name"] }}:
+    .. _config:{{ section_name }}:
 
-    [{{ section["name"] }}]
-    {{ "=" * (section["name"]|length + 2) }}
+    [{{ section_name }}]
+    {{ "=" * (section_name|length + 2) }}
 
     {% if 'renamed' in section %}
     *Renamed in version {{ section['renamed']['version'] }}, previous name was {{ section['renamed']['previous_name'] }}*
@@ -56,12 +56,12 @@ that you run airflow components on is synchronized (for example using ntpd) othe
     {{ section["description"] }}
     {% endif %}
 
-    {% for option in section["options"] %}
+    {% for option_name, option in section["options"].items() %}
 
-    .. _config:{{ section["name"] }}__{{ option["name"] }}:
+    .. _config:{{ section_name }}__{{ option_name }}:
 
-    {{ option["name"] }}
-    {{ "-" * option["name"]|length }}
+    {{ option_name }}
+    {{ "-" * option_name|length }}
 
     {% if option["version_added"] %}
     .. versionadded:: {{ option["version_added"] }}
@@ -79,13 +79,13 @@ that you run airflow components on is synchronized (for example using ntpd) othe
     :Default: ``{{ "''" if option["default"] == "" else option["default"] }}``
     {% if option.get("sensitive") %}
     :Environment Variables:
-        ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}``
+        ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}``
 
-        ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}_CMD``
+        ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_CMD``
 
-        ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}_SECRET``
+        ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_SECRET``
     {% else %}
-    :Environment Variable: ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}``
+    :Environment Variable: ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}``
     {% endif %}
     {% if option["example"] %}
     :Example:
@@ -94,10 +94,10 @@ that you run airflow components on is synchronized (for example using ntpd) othe
 
     {% endfor %}
 
-    {% if section["name"] in deprecated_options %}
+    {% if section_name in deprecated_options %}
 
-    {% for deprecated_option_name, (new_section_name, new_option_name, since_version) in deprecated_options[section["name"]].items() %}
-    .. _config:{{ section["name"] }}__{{ deprecated_option_name }}:
+    {% for deprecated_option_name, (new_section_name, new_option_name, since_version) in deprecated_options[section_name].items() %}
+    .. _config:{{ section_name }}__{{ deprecated_option_name }}:
 
     {{ deprecated_option_name }} (Deprecated)
     {{ "-" * (deprecated_option_name + " (Deprecated)")|length }}

docs/conf.py

@@ -397,15 +397,15 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path):
     # the config has been templated, not before
     # e.g. {{dag_id}} in default_config.cfg -> {dag_id} in airflow.cfg, and what we want in docs
     keys_to_format = ["default", "example"]
-    for conf_section in configs:
-        for option in conf_section["options"]:
+    for conf_name, conf_section in configs.items():
+        for option_name, option in conf_section["options"].items():
             for key in keys_to_format:
                 if option[key] and "{{" in option[key]:
                     option[key] = option[key].replace("{{", "{").replace("}}", "}")
     # Sort options, config and deprecated options for JINJA variables to display
-    for config in configs:
-        config["options"] = sorted(config["options"], key=lambda o: o["name"])
-    configs = sorted(configs, key=lambda l: l["name"])
+    for section_name, config in configs.items():
+        config["options"] = {k: v for k, v in sorted(config["options"].items())}
+    configs = {k: v for k, v in sorted(configs.items())}
     for section in deprecated_options:
         deprecated_options[section] = {k: v for k, v in sorted(deprecated_options[section].items())}
 

scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py

@@ -81,12 +81,11 @@ def write_config(yaml_config_file_path: str, default_cfg_file_path: str):
         configfile.writelines(FILE_HEADER)
         config_yaml = read_default_config_yaml(yaml_config_file_path)
 
-        for section in config_yaml:
-            _write_section(configfile, section)
+        for section_name, section in config_yaml.items():
+            _write_section(configfile, section_name, section)
 
 
-def _write_section(configfile, section):
-    section_name = section["name"]
+def _write_section(configfile, section_name, section):
     configfile.write(f"\n[{section_name}]\n")
     section_description = None
     if section["description"] is not None:
@@ -100,11 +99,11 @@ def _write_section(configfile, section):
                 configfile.write("#\n")
             else:
                 configfile.write(f"# {single_line_desc}\n")
-    for idx, option in enumerate(section["options"]):
-        _write_option(configfile, idx, option)
+    for idx, (option_name, option) in enumerate(section["options"].items()):
+        _write_option(configfile, idx, option_name, option)
 
 
-def _write_option(configfile, idx, option):
+def _write_option(configfile, idx, option_name, option):
     option_description = None
     if option["description"] is not None:
         option_description = list(filter(lambda x: x is not None, option["description"].splitlines()))
@@ -119,24 +118,24 @@ def _write_option(configfile, idx, option):
                 configfile.write(f"# {single_line_desc}\n")
 
     if option["example"]:
-        if not str(option["name"]).endswith("_template"):
+        if not str(option_name).endswith("_template"):
             option["example"] = option["example"].replace("{", "{{").replace("}", "}}")
-        configfile.write(f"# Example: {option['name']} = {option['example']}\n")
+        configfile.write(f"# Example: {option_name} = {option['example']}\n")
 
     if option["default"] is not None:
         if not isinstance(option["default"], str):
             raise Exception(
-                f"Key \"default\" in element with name=\"{option['name']}\" has an invalid type. "
+                f'Key "default" in element with name="{option_name}" has an invalid type. '
                 f"Current type: {type(option['default'])}"
             )
         # Remove trailing whitespace on empty string
         if option["default"]:
             value = " " + option["default"]
         else:
             value = ""
-        configfile.write(f"{option['name']} ={value}\n")
+        configfile.write(f"{option_name} ={value}\n")
     else:
-        configfile.write(f"# {option['name']} =\n")
+        configfile.write(f"# {option_name} =\n")
 
 
 if __name__ == "__main__":