Skip to content

Commit

Permalink
🐛 Fix config embed and restore (#27628)
Browse files Browse the repository at this point in the history
Co-authored-by: Scott Lahteine <[email protected]>
  • Loading branch information
ellensp and thinkyhead authored Jan 8, 2025
1 parent 83278bd commit 36623a2
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 71 deletions.
18 changes: 12 additions & 6 deletions buildroot/bin/config.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
'''
config.py - Helper functions for config manipulation
Make sure both copies always match:
- buildroot/bin/config.py
- buildroot/share/PlatformIO/scripts/config.py
'''
import re

Expand All @@ -17,24 +22,25 @@ def set(file_path, define_name, value):
modified = False
for i in range(len(content)):
# Regex to match the desired pattern
match = re.match(r'^(\s*)(/*)(\s*)(#define\s+{})\s+(.*)$'.format(re.escape(define_name)), content[i])
match = re.match(r'^(\s*)(/*)(\s*)(#define\s+{})\s+(.*?)\s*(//.*)?$'.format(re.escape(define_name)), content[i])
if match:
new_line = f"{match[1]}{match[3]}{match[4]} {value} // {match[5]}\n"
content[i] = new_line
modified = True
comm = '' if match[6] is None else ' ' + match[6]
oldval = '' if match[5] is None else match[5]
if match[2] or value != oldval:
content[i] = f"{match[1]}{match[3]}{match[4]} {value} // {match[5]}{comm}\n"

# Write the modified content back to the file only if changes were made
if modified:
with open(file_path, 'w') as f:
f.writelines(content)
return True
return True

return False

def add(file_path, define_name, value=""):
'''
Insert a define on the first blank line in a file.
Returns True if the define was found and replaced, False otherwise.
'''
with open(file_path, 'r') as f:
content = f.readlines()
Expand Down Expand Up @@ -66,7 +72,7 @@ def enable(file_path, define_name, enable=True):
content = f.readlines()

# Prepare the regex
regex = re.compile(r'^(\s*)(/*)(\s*)(#define\s+{}\b.*?)( *//.*)?$'.format(re.escape(define_name)))
regex = re.compile(r'^(\s*)(/*)(\s*)(#define\s+{}\b.*?)(\s*//.*)?$'.format(re.escape(define_name)))

# Find the define in the file and uncomment or comment it
found = False
Expand Down
102 changes: 102 additions & 0 deletions buildroot/share/PlatformIO/scripts/config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
'''
config.py - Helper functions for config manipulation
Make sure both copies always match:
- buildroot/bin/config.py
- buildroot/share/PlatformIO/scripts/config.py
'''
import re

FILES = ('Marlin/Configuration.h', 'Marlin/Configuration_adv.h')

def set(file_path, define_name, value):
'''
Replaces a define in a file with a new value.
Returns True if the define was found and replaced, False otherwise.
'''
# Read the contents of the file
with open(file_path, 'r') as f:
content = f.readlines()

modified = False
for i in range(len(content)):
# Regex to match the desired pattern
match = re.match(r'^(\s*)(/*)(\s*)(#define\s+{})\s+(.*?)\s*(//.*)?$'.format(re.escape(define_name)), content[i])
if match:
modified = True
comm = '' if match[6] is None else ' ' + match[6]
oldval = '' if match[5] is None else match[5]
if match[2] or value != oldval:
content[i] = f"{match[1]}{match[3]}{match[4]} {value} // {match[5]}{comm}\n"

# Write the modified content back to the file only if changes were made
if modified:
with open(file_path, 'w') as f:
f.writelines(content)
return True

return False

def add(file_path, define_name, value=""):
'''
Insert a define on the first blank line in a file.
'''
with open(file_path, 'r') as f:
content = f.readlines()

# Prepend a space to the value if it's not empty
if value != "":
value = " " + value

# Find the first blank line to insert the new define
for i in range(len(content)):
if content[i].strip() == '':
# Insert the define at the first blank line
content.insert(i, f"#define {define_name}{value}\n")
break
else:
# If no blank line is found, append to the end
content.append(f"#define {define_name}{value}\n")

with open(file_path, 'w') as f:
f.writelines(content)

def enable(file_path, define_name, enable=True):
'''
Uncomment or comment the named defines in the given file path.
Returns True if the define was found, False otherwise.
'''
# Read the contents of the file
with open(file_path, 'r') as f:
content = f.readlines()

# Prepare the regex
regex = re.compile(r'^(\s*)(/*)(\s*)(#define\s+{}\b.*?)(\s*//.*)?$'.format(re.escape(define_name)))

# Find the define in the file and uncomment or comment it
found = False
modified = False
for i in range(len(content)):
match = regex.match(content[i])
if not match: continue
found = True
if enable:
if match[2]:
modified = True
comment = '' if match[5] is None else ' ' + match[5]
content[i] = f"{match[1]}{match[3]}{match[4]}{comment}\n"
else:
if not match[2]:
modified = True
comment = '' if match[5] is None else match[5]
if comment.startswith(' '): comment = comment[2:]
content[i] = f"{match[1]}//{match[3]}{match[4]}{comment}\n"
break

# Write the modified content back to the file only if changes were made
if modified:
with open(file_path, 'w') as f:
f.writelines(content)

return found
152 changes: 93 additions & 59 deletions buildroot/share/PlatformIO/scripts/mc-apply.py
Original file line number Diff line number Diff line change
@@ -1,66 +1,100 @@
#!/usr/bin/env python
#
# Create a Configuration from marlin_config.json
# mc-apply.py
#
import json, sys, shutil
# Apply firmware configuration from a JSON file (marlin_config.json).
#
# usage: mc-apply.py [-h] [--opt] [config_file]
#
# Process Marlin firmware configuration.
#
# positional arguments:
# config_file Path to the configuration file.
#
# optional arguments:
# -h, --help show this help message and exit
# --opt Output as an option setting script.
#
import json, sys, os
import config
import argparse

opt_output = '--opt' in sys.argv
output_suffix = '.sh' if opt_output else '' if '--bare-output' in sys.argv else '.gen'
def report_version(conf):
if 'VERSION' in conf:
for k, v in sorted(conf['VERSION'].items()):
print(k + ': ' + v)

try:
with open('marlin_config.json', 'r') as infile:
conf = json.load(infile)
for key in conf:
# We don't care about the hash when restoring here
if key == '__INITIAL_HASH':
continue
if key == 'VERSION':
for k, v in sorted(conf[key].items()):
print(k + ': ' + v)
def write_opt_file(conf, outpath='Marlin/apply_config.sh'):
with open(outpath, 'w') as outfile:
for key, val in conf.items():
if key in ('__INITIAL_HASH', 'VERSION'): continue

# Other keys are assumed to be configs
if not type(val) is dict:
continue
# The key is the file name, so let's build it now
outfile = open('Marlin/' + key + output_suffix, 'w')
for k, v in sorted(conf[key].items()):
# Make define line now
if opt_output:
if v != '':
if '"' in v:
v = "'%s'" % v
elif ' ' in v:
v = '"%s"' % v
define = 'opt_set ' + k + ' ' + v + '\n'
else:
define = 'opt_enable ' + k + '\n'

# Write config commands to the script file
lines = []
for k, v in sorted(val.items()):
if v != '':
v.replace('"', '\\"').replace("'", "\\'").replace(' ', '\\ ')
lines += [f'opt_set {k} {v}']
else:
define = '#define ' + k + ' ' + v + '\n'
outfile.write(define)
outfile.close()

# Try to apply changes to the actual configuration file (in order to keep useful comments)
if output_suffix != '':
# Move the existing configuration so it doesn't interfere
shutil.move('Marlin/' + key, 'Marlin/' + key + '.orig')
infile_lines = open('Marlin/' + key + '.orig', 'r').read().split('\n')
outfile = open('Marlin/' + key, 'w')
for line in infile_lines:
sline = line.strip(" \t\n\r")
if sline[:7] == "#define":
# Extract the key here (we don't care about the value)
kv = sline[8:].strip().split(' ')
if kv[0] in conf[key]:
outfile.write('#define ' + kv[0] + ' ' + conf[key][kv[0]] + '\n')
# Remove the key from the dict, so we can still write all missing keys at the end of the file
del conf[key][kv[0]]
else:
outfile.write(line + '\n')
else:
outfile.write(line + '\n')
# Process any remaining defines here
for k, v in sorted(conf[key].items()):
define = '#define ' + k + ' ' + v + '\n'
outfile.write(define)
outfile.close()

print('Output configuration written to: ' + 'Marlin/' + key + output_suffix)
except:
print('No marlin_config.json found.')
lines += [f'opt_enable {k}']

outfile.write('\n'.join(lines))

print('Config script written to: ' + outpath)

def back_up_config(name):
# Back up the existing file before modifying it
conf_path = 'Marlin/' + name
with open(conf_path, 'r') as f:
# Write a filename.bak#.ext retaining the original extension
parts = conf_path.split('.')
nr = ''
while True:
bak_path = '.'.join(parts[:-1]) + f'.bak{nr}.' + parts[-1]
if os.path.exists(bak_path):
nr = 1 if nr == '' else nr + 1
continue

with open(bak_path, 'w') as b:
b.writelines(f.readlines())
break

def apply_config(conf):
for key in conf:
if key in ('__INITIAL_HASH', 'VERSION'): continue

back_up_config(key)

for k, v in conf[key].items():
if v:
config.set('Marlin/' + key, k, v)
else:
config.enable('Marlin/' + key, k)

def main():
parser = argparse.ArgumentParser(description='Process Marlin firmware configuration.')
parser.add_argument('--opt', action='store_true', help='Output as an option setting script.')
parser.add_argument('config_file', nargs='?', default='marlin_config.json', help='Path to the configuration file.')

args = parser.parse_args()

try:
infile = open(args.config_file, 'r')
except:
print(f'No {args.config_file} found.')
sys.exit(1)

conf = json.load(infile)
report_version(conf)

if args.opt:
write_opt_file(conf)
else:
apply_config(conf)

if __name__ == '__main__':
main()
16 changes: 10 additions & 6 deletions buildroot/share/PlatformIO/scripts/signature.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ def get_file_sha256sum(filepath):
#
import zipfile
def compress_file(filepath, storedname, outpath):
with zipfile.ZipFile(outpath, 'w', compression=zipfile.ZIP_BZIP2, compresslevel=9) as zipf:
zipf.write(filepath, arcname=storedname, compress_type=zipfile.ZIP_BZIP2, compresslevel=9)
with zipfile.ZipFile(outpath, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=False, compresslevel=9) as zipf:
zipf.write(filepath, arcname=storedname)

ignore = ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION', 'CONFIG_EXAMPLES_DIR', 'CONFIG_EXPORT')

Expand Down Expand Up @@ -161,7 +161,8 @@ def compute_build_signature(env):
#
# Continue to gather data for CONFIGURATION_EMBEDDING or CONFIG_EXPORT
#
if not ('CONFIGURATION_EMBEDDING' in build_defines or 'CONFIG_EXPORT' in build_defines):
is_embed = 'CONFIGURATION_EMBEDDING' in build_defines
if not (is_embed or 'CONFIG_EXPORT' in build_defines):
return

# Filter out useless macros from the output
Expand Down Expand Up @@ -450,7 +451,7 @@ def optsort(x, optorder):
# Produce a JSON file for CONFIGURATION_EMBEDDING or CONFIG_EXPORT == 1 or 101
# Skip if an identical JSON file was already present.
#
if not same_hash and (config_dump == 1 or 'CONFIGURATION_EMBEDDING' in build_defines):
if not same_hash and (config_dump == 1 or is_embed):
with marlin_json.open('w') as outfile:

json_data = {}
Expand All @@ -460,16 +461,19 @@ def optsort(x, optorder):
confs = real_config[header]
json_data[header] = {}
for name in confs:
if name in ignore: continue
c = confs[name]
s = c['section']
if s not in json_data[header]: json_data[header][s] = {}
json_data[header][s][name] = c['value']
else:
for header in real_config:
json_data[header] = {}
conf = real_config[header]
#print(f"real_config[{header}]", conf)
for name in conf:
json_data[name] = conf[name]['value']
if name in ignore: continue
json_data[header][name] = conf[name]['value']

json_data['__INITIAL_HASH'] = hashes

Expand All @@ -489,7 +493,7 @@ def optsort(x, optorder):
#
# The rest only applies to CONFIGURATION_EMBEDDING
#
if not 'CONFIGURATION_EMBEDDING' in build_defines:
if not is_embed:
(build_path / 'mc.zip').unlink(missing_ok=True)
return

Expand Down

0 comments on commit 36623a2

Please sign in to comment.