🔨 Update config/schema scripts (MarlinFirmware#26483)

Marlin/config.ini

@@ -3,10 +3,50 @@
 # config.ini - Options to apply before the build
 #
 [config:base]
+#
+# ini_use_config - A comma-separated list of actions to apply to the Configuration files.
+#                  The actions will be applied in the listed order.
+#  - none
+#    Ignore this file and don't apply any configuration options
+#
+#  - base
+#    Just apply the options in config:base to the configuration
+#
+#  - minimal
+#    Just apply the options in config:minimal to the configuration
+#
+#  - all
+#    Apply all 'config:*' sections in this file to the configuration
+#
+#  - another.ini
+#    Load another INI file with a path relative to this config.ini file (i.e., within Marlin/)
+#
+#  - https://me.myserver.com/path/to/configs
+#    Fetch configurations from any URL.
+#
+#  - example/Creality/Ender-5 Plus @ bugfix-2.1.x
+#    Fetch example configuration files from the MarlinFirmware/Configurations repository
+#    https://raw.githubusercontent.com/MarlinFirmware/Configurations/bugfix-2.1.x/config/examples/Creality/Ender-5%20Plus/
+#
+#  - example/default @ release-2.0.9.7
+#    Fetch default configuration files from the MarlinFirmware/Configurations repository
+#    https://raw.githubusercontent.com/MarlinFirmware/Configurations/release-2.0.9.7/config/default/
+#
+#  - [disable]
+#    Comment out all #defines in both Configuration.h and Configuration_adv.h. This is useful
+#    to start with a clean slate before applying any config: options, so only the options explicitly
+#    set in config.ini will be enabled in the configuration.
+#
+#  - [flatten] (Not yet implemented)
+#    Produce a flattened set of Configuration.h and Configuration_adv.h files with only the enabled
+#    #defines and no comments. A clean look, but context-free.
+#
 ini_use_config                           = none
 
 # Load all config: sections in this file
 ;ini_use_config                          = all
+# Disable everything and apply subsequent config:base options
+;ini_use_config                          = [disable], base
 # Load config file relative to Marlin/
 ;ini_use_config                          = another.ini
 # Download configurations from GitHub

buildroot/share/PlatformIO/scripts/configuration.py

@@ -1,8 +1,9 @@
+#!/usr/bin/env python3
 #
 # configuration.py
 # Apply options from config.ini to the existing Configuration headers
 #
-import re, shutil, configparser
+import re, shutil, configparser, datetime
 from pathlib import Path
 
 verbose = 0
@@ -43,6 +44,7 @@ def apply_opt(name, val, conf=None):
                 if val in ("on", "", None):
                     newline = re.sub(r'^(\s*)//+\s*(#define)(\s{1,3})?(\s*)', r'\1\2 \4', line)
                 elif val == "off":
+                    # TODO: Comment more lines in a multi-line define with \ continuation
                     newline = re.sub(r'^(\s*)(#define)(\s{1,3})?(\s*)', r'\1//\2 \4', line)
                 else:
                     # For options with values, enable and set the value
@@ -88,9 +90,38 @@ def apply_opt(name, val, conf=None):
                 elif not isdef:
                     break
                 linenum += 1
-            lines.insert(linenum, f"{prefix}#define {added:30} // Added by config.ini\n")
+            currtime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            lines.insert(linenum, f"{prefix}#define {added:30} // Added by config.ini {currtime}\n")
             fullpath.write_text(''.join(lines), encoding='utf-8')
 
+# Disable all (most) defined options in the configuration files.
+# Everything in the named sections. Section hint for exceptions may be added.
+def disable_all_options():
+    # Create a regex to match the option and capture parts of the line
+    regex = re.compile(r'^(\s*)(#define\s+)([A-Z0-9_]+\b)(\s?)(\s*)(.*?)(\s*)(//.*)?$', re.IGNORECASE)
+
+    # Disable all enabled options in both Config files
+    for file in ("Configuration.h", "Configuration_adv.h"):
+        fullpath = config_path(file)
+        lines = fullpath.read_text(encoding='utf-8').split('\n')
+        found = False
+        for i in range(len(lines)):
+            line = lines[i]
+            match = regex.match(line)
+            if match:
+                name = match[3].upper()
+                if name in ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION'): continue
+                if name.startswith('_'): continue
+                found = True
+                # Comment out the define
+                # TODO: Comment more lines in a multi-line define with \ continuation
+                lines[i] = re.sub(r'^(\s*)(#define)(\s{1,3})?(\s*)', r'\1//\2 \4', line)
+                blab(f"Disable {name}")
+
+        # If the option was found, write the modified lines
+        if found:
+            fullpath.write_text('\n'.join(lines), encoding='utf-8')
+
 # Fetch configuration files from GitHub given the path.
 # Return True if any files were fetched.
 def fetch_example(url):
@@ -130,7 +161,7 @@ def fetch_example(url):
 def section_items(cp, sectkey):
     return cp.items(sectkey) if sectkey in cp.sections() else []
 
-# Apply all items from a config section
+# Apply all items from a config section. Ignore ini_ items outside of config:base and config:root.
 def apply_ini_by_name(cp, sect):
     iniok = True
     if sect in ('config:base', 'config:root'):
@@ -206,7 +237,17 @@ def apply_config_ini(cp):
             fetch_example(ckey)
             ckey = 'base'
 
-        if ckey == 'all':
+        #
+        # [flatten] Write out Configuration.h and Configuration_adv.h files with
+        #           just the enabled options and all other content removed.
+        #
+        #if ckey == '[flatten]':
+        #   write_flat_configs()
+
+        if ckey == '[disable]':
+            disable_all_options()
+
+        elif ckey == 'all':
             apply_sections(cp)
 
         else:

buildroot/share/PlatformIO/scripts/schema.py

@@ -2,8 +2,14 @@
 #
 # schema.py
 #
-# Used by signature.py via common-dependencies.py to generate a schema file during the PlatformIO build.
-# This script can also be run standalone from within the Marlin repo to generate all schema files.
+# Used by signature.py via common-dependencies.py to generate a schema file during the PlatformIO build
+# when CONFIG_EXPORT is defined in the configuration.
+#
+# This script can also be run standalone from within the Marlin repo to generate JSON and YAML schema files.
+#
+# This script is a companion to abm/js/schema.js in the MarlinFirmware/AutoBuildMarlin project, which has
+# been extended to evaluate conditions and can determine what options are actually enabled, not just which
+# options are uncommented. That will be migrated to this script for standalone migration.
 #
 import re,json
 from pathlib import Path
@@ -95,6 +101,8 @@ class Parse:
     sch_out = { 'basic':{}, 'advanced':{} }
     # Regex for #define NAME [VALUE] [COMMENT] with sanitized line
     defgrep = re.compile(r'^(//)?\s*(#define)\s+([A-Za-z0-9_]+)\s*(.*?)\s*(//.+)?$')
+    # Pattern to match a float value
+    flt = r'[-+]?\s*(\d+\.|\d*\.\d+)([eE][-+]?\d+)?[fF]?'
     # Defines to ignore
     ignore = ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION', 'CONFIG_EXAMPLES_DIR', 'CONFIG_EXPORT')
     # Start with unknown state
@@ -314,26 +322,27 @@ def atomize(s):
                         }
 
                         # Type is based on the value
-                        if val == '':
-                            value_type = 'switch'
-                        elif re.match(r'^(true|false)$', val):
-                            value_type = 'bool'
-                            val = val == 'true'
-                        elif re.match(r'^[-+]?\s*\d+$', val):
-                            value_type = 'int'
-                            val = int(val)
-                        elif re.match(r'[-+]?\s*(\d+\.|\d*\.\d+)([eE][-+]?\d+)?[fF]?', val):
-                            value_type = 'float'
-                            val = float(val.replace('f',''))
-                        else:
-                            value_type = 'string'   if val[0] == '"' \
-                                    else 'char'     if val[0] == "'" \
-                                    else 'state'    if re.match(r'^(LOW|HIGH)$', val) \
-                                    else 'enum'     if re.match(r'^[A-Za-z0-9_]{3,}$', val) \
-                                    else 'int[]'    if re.match(r'^{(\s*[-+]?\s*\d+\s*(,\s*)?)+}$', val) \
-                                    else 'float[]'  if re.match(r'^{(\s*[-+]?\s*(\d+\.|\d*\.\d+)([eE][-+]?\d+)?[fF]?\s*(,\s*)?)+}$', val) \
-                                    else 'array'    if val[0] == '{' \
-                                    else ''
+                        value_type = \
+                             'switch'  if val == '' \
+                        else 'bool'    if re.match(r'^(true|false)$', val) \
+                        else 'int'     if re.match(r'^[-+]?\s*\d+$', val) \
+                        else 'ints'    if re.match(r'^([-+]?\s*\d+)(\s*,\s*[-+]?\s*\d+)+$', val) \
+                        else 'floats'  if re.match(rf'({flt}(\s*,\s*{flt})+)', val) \
+                        else 'float'   if re.match(f'^({flt})$', val) \
+                        else 'string'  if val[0] == '"' \
+                        else 'char'    if val[0] == "'" \
+                        else 'state'   if re.match(r'^(LOW|HIGH)$', val) \
+                        else 'enum'    if re.match(r'^[A-Za-z0-9_]{3,}$', val) \
+                        else 'int[]'   if re.match(r'^{\s*[-+]?\s*\d+(\s*,\s*[-+]?\s*\d+)*\s*}$', val) \
+                        else 'float[]' if re.match(r'^{{\s*{flt}(\s*,\s*{flt})*\s*}}$', val) \
+                        else 'array'   if val[0] == '{' \
+                        else ''
+
+                        val = (val == 'true')           if value_type == 'bool' \
+                        else int(val)                   if value_type == 'int' \
+                        else val.replace('f','')        if value_type == 'floats' \
+                        else float(val.replace('f','')) if value_type == 'float' \
+                        else val
 
                         if val != '': define_info['value'] = val
                         if value_type != '': define_info['type'] = value_type
@@ -402,25 +411,35 @@ def main():
 
     if schema:
 
-        # Get the first command line argument
+        # Get the command line arguments after the script name
         import sys
-        if len(sys.argv) > 1:
-            arg = sys.argv[1]
-        else:
-            arg = 'some'
+        args = sys.argv[1:]
+        if len(args) == 0: args = ['some']
+
+        # Does the given array intersect at all with args?
+        def inargs(c): return len(set(args) & set(c)) > 0
+
+        # Help / Unknown option
+        unk = not inargs(['some','json','jsons','group','yml','yaml'])
+        if (unk): print(f"Unknown option: '{args[0]}'")
+        if inargs(['-h', '--help']) or unk:
+            print("Usage: schema.py [some|json|jsons|group|yml|yaml]...")
+            print("       some  = json + yml")
+            print("       jsons = json + group")
+            return
 
         # JSON schema
-        if arg in ['some', 'json', 'jsons']:
+        if inargs(['some', 'json', 'jsons']):
             print("Generating JSON ...")
             dump_json(schema, Path('schema.json'))
 
         # JSON schema (wildcard names)
-        if arg in ['group', 'jsons']:
+        if inargs(['group', 'jsons']):
             group_options(schema)
             dump_json(schema, Path('schema_grouped.json'))
 
         # YAML
-        if arg in ['some', 'yml', 'yaml']:
+        if inargs(['some', 'yml', 'yaml']):
             try:
                 import yaml
             except ImportError:

buildroot/share/PlatformIO/scripts/signature.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 #
 # signature.py
 #
@@ -44,35 +45,35 @@ def compress_file(filepath, storedname, outpath):
         zipf.write(filepath, arcname=storedname, compress_type=zipfile.ZIP_BZIP2, compresslevel=9)
 
 #
-# Compute the build signature. The idea is to extract all defines in the configuration headers
-# to build a unique reversible signature from this build so it can be included in the binary
-# We can reverse the signature to get a 1:1 equivalent configuration file
+# Compute the build signature by extracting all configuration settings and
+# building a unique reversible signature that can be included in the binary.
+# The signature can be reversed to get a 1:1 equivalent configuration file.
 #
 def compute_build_signature(env):
     if 'BUILD_SIGNATURE' in env:
         return
 
+    build_path = Path(env['PROJECT_BUILD_DIR'], env['PIOENV'])
+    marlin_json = build_path / 'marlin_config.json'
+    marlin_zip = build_path / 'mc.zip'
+
     # Definitions from these files will be kept
     files_to_keep = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ]
 
-    build_path = Path(env['PROJECT_BUILD_DIR'], env['PIOENV'])
-
     # Check if we can skip processing
     hashes = ''
     for header in files_to_keep:
         hashes += get_file_sha256sum(header)[0:10]
 
-    marlin_json = build_path / 'marlin_config.json'
-    marlin_zip = build_path / 'mc.zip'
-
-    # Read existing config file
+    # Read a previously exported JSON file
+    # Same configuration, skip recomputing the build signature
+    same_hash = False
     try:
         with marlin_json.open() as infile:
             conf = json.load(infile)
-            if conf['__INITIAL_HASH'] == hashes:
-                # Same configuration, skip recomputing the building signature
+            same_hash = conf['__INITIAL_HASH'] == hashes
+            if same_hash:
                 compress_file(marlin_json, 'marlin_config.json', marlin_zip)
-                return
     except:
         pass
 
@@ -125,9 +126,6 @@ def compute_build_signature(env):
         # Remove all boards now
         if key.startswith("BOARD_") and key != "BOARD_INFO_NAME":
             continue
-        # Remove all keys ending by "_NAME" as it does not make a difference to the configuration
-        if key.endswith("_NAME") and key != "CUSTOM_MACHINE_NAME":
-            continue
         # Remove all keys ending by "_T_DECLARED" as it's a copy of extraneous system stuff
         if key.endswith("_T_DECLARED"):
             continue
@@ -196,7 +194,7 @@ def tryint(key):
                         outfile.write(ini_fmt.format(key.lower(), ' = ' + val))
 
     #
-    # Produce a schema.json file if CONFIG_EXPORT == 3
+    # CONFIG_EXPORT 3 = schema.json, 4 = schema.yml
     #
     if config_dump >= 3:
         try:
@@ -207,7 +205,7 @@ def tryint(key):
 
         if conf_schema:
             #
-            # Produce a schema.json file if CONFIG_EXPORT == 3
+            # 3 = schema.json
             #
             if config_dump in (3, 13):
                 print("Generating schema.json ...")
@@ -217,7 +215,7 @@ def tryint(key):
                     schema.dump_json(conf_schema, build_path / 'schema_grouped.json')
 
             #
-            # Produce a schema.yml file if CONFIG_EXPORT == 4
+            # 4 = schema.yml
             #
             elif config_dump == 4:
                 print("Generating schema.yml ...")
@@ -243,8 +241,9 @@ def tryint(key):
 
     #
     # Produce a JSON file for CONFIGURATION_EMBEDDING or CONFIG_EXPORT == 1
+    # Skip if an identical JSON file was already present.
     #
-    if config_dump == 1 or 'CONFIGURATION_EMBEDDING' in defines:
+    if not same_hash and (config_dump == 1 or 'CONFIGURATION_EMBEDDING' in defines):
         with marlin_json.open('w') as outfile:
             json.dump(data, outfile, separators=(',', ':'))
 
@@ -255,9 +254,10 @@ def tryint(key):
         return
 
     # Compress the JSON file as much as we can
-    compress_file(marlin_json, 'marlin_config.json', marlin_zip)
+    if not same_hash:
+        compress_file(marlin_json, 'marlin_config.json', marlin_zip)
 
-    # Generate a C source file for storing this array
+    # Generate a C source file containing the entire ZIP file as an array
     with open('Marlin/src/mczip.h','wb') as result_file:
         result_file.write(
               b'#ifndef NO_CONFIGURATION_EMBEDDING_WARNING\n'
@@ -274,3 +274,8 @@ def tryint(key):
         if count % 16:
             result_file.write(b'\n')
         result_file.write(b'};\n')
+
+if __name__ == "__main__":
+    # Build required. From command line just explain usage.
+    print("Use schema.py to export JSON and YAML from the command-line.")
+    print("Build Marlin with CONFIG_EXPORT 2 to export 'config.ini'.")