Use pathlib everywhere we can (qmk#7872)

* Use pathlib everywhere we can

* Update lib/python/qmk/path.py

Co-Authored-By: Erovia <[email protected]>

* Update lib/python/qmk/path.py

Co-Authored-By: Erovia <[email protected]>

* Improvements based on @Erovia's feedback

* rework qmk compile and qmk flash to use pathlib

* style

* Remove the subcommand_name argument from find_keyboard_keymap()

Co-authored-by: Erovia <[email protected]>

lib/python/milc.py

@@ -273,7 +273,7 @@ def __init__(self):
         self._inside_context_manager = False
         self.ansi = ansi_colors
         self.arg_only = []
-        self.config = None
+        self.config = self.config_source = None
         self.config_file = None
         self.default_arguments = {}
         self.version = 'unknown'
@@ -473,6 +473,7 @@ def read_config_file(self):
         """
         self.acquire_lock()
         self.config = Configuration()
+        self.config_source = Configuration()
         self.config_file = self.find_config_file()
 
         if self.config_file and self.config_file.exists():
@@ -498,6 +499,7 @@ def read_config_file(self):
                             value = int(value)
 
                     self.config[section][option] = value
+                    self.config_source[section][option] = 'config_file'
 
         self.release_lock()
 
@@ -530,12 +532,14 @@ def merge_args_into_config(self):
                     arg_value = getattr(self.args, argument)
                     if arg_value is not None:
                         self.config[section][argument] = arg_value
+                        self.config_source[section][argument] = 'argument'
                 else:
                     if argument not in self.config[entrypoint_name]:
                         # Check if the argument exist for this section
                         arg = getattr(self.args, argument)
                         if arg is not None:
                             self.config[section][argument] = arg
+                            self.config_source[section][argument] = 'argument'
 
         self.release_lock()
 

lib/python/qmk/cli/compile.py

@@ -3,16 +3,12 @@
 You can compile a keymap already in the repo or using a QMK Configurator export.
 """
 import subprocess
-import os
 from argparse import FileType
 
 from milc import cli
-from qmk.commands import create_make_command
-from qmk.commands import parse_configurator_json
-from qmk.commands import compile_configurator_json
 
-import qmk.keymap
 import qmk.path
+from qmk.commands import compile_configurator_json, create_make_command, find_keyboard_keymap, parse_configurator_json
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export to compile')
@@ -24,99 +20,28 @@ def compile(cli):
 
     If a Configurator export is supplied this command will create a new keymap, overwriting an existing keymap if one exists.
 
-    FIXME(skullydazed): add code to check and warn if the keymap already exists
-
-    If --keyboard and --keymap are provided this command will build a firmware based on that.
-
+    If a keyboard and keymap are provided this command will build a firmware based on that.
     """
-    # Set CWD as directory command was issued from
-    cwd = os.environ['ORIG_CWD']
-    qmk_path = os.getcwd()
-    current_folder = os.path.basename(cwd)
-    # Initialize boolean to check for being in a keyboard directory and initialize keyboard string
-    in_keyboard = False
-    in_layout = False
-    keyboard = ""
-    keymap = ""
-    user_keymap = ""
-    user_keyboard = ""
-
-    # Set path for '/keyboards/' directory
-    keyboards_path = os.path.join(qmk_path, "keyboards")
-    layouts_path = os.path.join(qmk_path, "layouts")
-
-    # If below 'keyboards' and not in 'keyboards' or 'keymaps', get current keyboard name
-    if cwd.startswith(keyboards_path):
-        if current_folder != "keyboards" and current_folder != "keymaps":
-            if os.path.basename(os.path.abspath(os.path.join(cwd, ".."))) == "keymaps":
-                # If in a keymap folder, set relative path, get everything before /keymaps, and the keymap name
-                relative_path = cwd[len(keyboards_path):][1:]
-                keyboard = str(relative_path).split("/keymaps", 1)[0]
-                keymap = str(relative_path.rsplit("/", 1)[-1])
-            else:
-                keyboard = str(cwd[len(keyboards_path):])[1:]
-
-            in_keyboard = True
-
-    # If in layouts dir
-    if cwd.startswith(layouts_path):
-        if current_folder != "layouts":
-            in_layout = True
-
-    # If user keyboard/keymap or compile keyboard/keymap are supplied, assign those
-    if cli.config.compile.keyboard:
-        user_keyboard = cli.config.compile.keyboard
-    if cli.config.compile.keymap and not in_layout:
-        user_keymap = cli.config.compile.keymap
-
     if cli.args.filename:
-        # Parse the configurator json
+        # If a configurator JSON was provided skip straight to compiling it
+        # FIXME(skullydazed): add code to check and warn if the keymap already exists when compiling a json keymap.
         user_keymap = parse_configurator_json(cli.args.filename)
-
-        # Generate the keymap
         keymap_path = qmk.path.keymap(user_keymap['keyboard'])
-        cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
-
-        # Compile the keymap
-        command = compile_configurator_json(cli.args.filename)
+        command = compile_configurator_json(user_keymap)
 
         cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
 
-    elif user_keyboard and user_keymap:
-        # Generate the make command for a specific keyboard/keymap.
-        command = create_make_command(user_keyboard, user_keymap)
-
-    elif in_keyboard:
-        keyboard = user_keyboard if user_keyboard else keyboard
-        keymap = user_keymap if user_keymap else keymap
-
-        if not os.path.exists(os.path.join(keyboards_path, keyboard, "rules.mk")):
-            cli.log.error('This directory does not contain a rules.mk file. Change directory or supply --keyboard with optional --keymap')
-            return False
-
-        # Get path for keyboard directory
-        keymap_path = qmk.path.keymap(keyboard)
-
-        # Check for global keymap config first
-        if keymap:
-            command = create_make_command(keyboard, keymap)
-
-        else:
-            # If no default keymap exists and none provided
-            cli.log.error('This directory does not contain a keymap. Set one with `qmk config` or supply `--keymap` ')
-            return False
+    else:
+        # Perform the action the user specified
+        user_keyboard, user_keymap = find_keyboard_keymap()
+        if user_keyboard and user_keymap:
+            # Generate the make command for a specific keyboard/keymap.
+            command = create_make_command(user_keyboard, user_keymap)
 
-    elif in_layout:
-        if user_keyboard:
-            keymap = current_folder
-            command = create_make_command(user_keyboard, keymap)
         else:
-            cli.log.error('You must supply a keyboard to compile a layout keymap. Set one with `qmk config` or supply `--keyboard` ')
+            cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
+            cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]')
             return False
 
-    else:
-        cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-        return False
-
     cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
     subprocess.run(command)

lib/python/qmk/cli/flash.py

@@ -7,7 +7,7 @@
 
 import qmk.path
 from milc import cli
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
+from qmk.commands import compile_configurator_json, create_make_command, find_keyboard_keymap, parse_configurator_json
 
 
 def print_bootloader_help():
@@ -44,39 +44,31 @@ def flash(cli):
     If bootloader is omitted, the one according to the rules.mk will be used.
 
     """
-    command = []
     if cli.args.bootloaders:
         # Provide usage and list bootloaders
         cli.echo('usage: qmk flash [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
         print_bootloader_help()
         return False
 
-    elif cli.args.keymap and not cli.args.keyboard:
-        # If only a keymap was given but no keyboard, suggest listing keyboards
-        cli.echo('usage: qmk flash [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
-        cli.log.error('run \'qmk list_keyboards\' to find out the supported keyboards')
-        return False
-
-    elif cli.args.filename:
-        # Get keymap path to log info
+    if cli.args.filename:
+        # Handle compiling a configurator JSON
         user_keymap = parse_configurator_json(cli.args.filename)
         keymap_path = qmk.path.keymap(user_keymap['keyboard'])
-
-        cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
-
-        # Convert the JSON into a C file and write it to disk.
-        command = compile_configurator_json(cli.args.filename, cli.args.bootloader)
+        command = compile_configurator_json(user_keymap, cli.args.bootloader)
 
         cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
 
-    elif cli.args.keyboard and cli.args.keymap:
-        # Generate the make command for a specific keyboard/keymap.
-        command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader)
-
     else:
-        cli.echo('usage: qmk flash [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
-        cli.log.error('You must supply a configurator export or both `--keyboard` and `--keymap`. You can also specify a bootloader with --bootloader. Use --bootloaders to list the available bootloaders.')
-        return False
+        # Perform the action the user specified
+        user_keyboard, user_keymap = find_keyboard_keymap()
+        if user_keyboard and user_keymap:
+            # Generate the make command for a specific keyboard/keymap.
+            command = create_make_command(user_keyboard, user_keymap, cli.args.bootloader)
+
+        else:
+            cli.log.error('You must supply a configurator export or both `--keyboard` and `--keymap`.')
+            cli.echo('usage: qmk flash [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
+            return False
 
     cli.log.info('Flashing keymap with {fg_cyan}%s\n\n', ' '.join(command))
     subprocess.run(command)

lib/python/qmk/cli/json/keymap.py

@@ -1,14 +1,15 @@
 """Generate a keymap.c from a configurator export.
 """
 import json
-import os
+from pathlib import Path
 
 from milc import cli
 
 import qmk.keymap
+import qmk.path
 
 
-@cli.argument('-o', '--output', arg_only=True, help='File to write to')
+@cli.argument('-o', '--output', arg_only=True, type=Path, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
 @cli.argument('filename', arg_only=True, help='Configurator JSON file')
 @cli.subcommand('Create a keymap.c from a QMK Configurator export.')
@@ -17,13 +18,17 @@ def json_keymap(cli):
 
     This command uses the `qmk.keymap` module to generate a keymap.c from a configurator export. The generated keymap is written to stdout, or to a file if -o is provided.
     """
+    cli.args.filename = qmk.path.normpath(cli.args.filename)
+
     # Error checking
-    if cli.args.filename == ('-'):
-        cli.log.error('Reading from STDIN is not (yet) supported.')
+    if not cli.args.filename.exists():
+        cli.log.error('JSON file does not exist!')
         cli.print_usage()
         exit(1)
-    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
-        cli.log.error('JSON file does not exist!')
+
+    if str(cli.args.filename) == '-':
+        # TODO(skullydazed/anyone): Read file contents from STDIN
+        cli.log.error('Reading from STDIN is not (yet) supported.')
         cli.print_usage()
         exit(1)
 
@@ -32,21 +37,17 @@ def json_keymap(cli):
         cli.args.output = None
 
     # Parse the configurator json
-    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+    with cli.args.filename.open('r') as fd:
         user_keymap = json.load(fd)
 
     # Generate the keymap
     keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
 
     if cli.args.output:
-        output_dir = os.path.dirname(cli.args.output)
-
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
-
-        output_file = qmk.path.normpath(cli.args.output)
-        with open(output_file, 'w') as keymap_fd:
-            keymap_fd.write(keymap_c)
+        cli.args.output.parent.mkdir(parents=True, exist_ok=True)
+        if cli.args.output.exists():
+            cli.args.output.replace(cli.args.output.name + '.bak')
+        cli.args.output.write_text(keymap_c)
 
         if not cli.args.quiet:
             cli.log.info('Wrote keymap to %s.', cli.args.output)

lib/python/qmk/cli/list/keyboards.py

@@ -1,5 +1,6 @@
 """List the keyboards currently defined within QMK
 """
+# We avoid pathlib here because this is performance critical code.
 import os
 import glob
 

lib/python/qmk/cli/new/keymap.py

@@ -1,8 +1,9 @@
 """This script automates the copying of the default keymap into your own keymap.
 """
-import os
 import shutil
+from pathlib import Path
 
+import qmk.path
 from milc import cli
 
 
@@ -17,24 +18,27 @@ def new_keymap(cli):
     keymap = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else input("Keymap Name: ")
 
     # generate keymap paths
-    kb_path = os.path.join(os.getcwd(), "keyboards", keyboard)
-    keymap_path_default = os.path.join(kb_path, "keymaps/default")
-    keymap_path = os.path.join(kb_path, "keymaps/%s" % keymap)
+    kb_path = Path('keyboards') / keyboard
+    keymap_path = qmk.path.keymap(keyboard)
+    keymap_path_default = keymap_path / 'default'
+    keymap_path_new = keymap_path / keymap
 
     # check directories
-    if not os.path.exists(kb_path):
+    if not kb_path.exists():
         cli.log.error('Keyboard %s does not exist!', kb_path)
         exit(1)
-    if not os.path.exists(keymap_path_default):
+
+    if not keymap_path_default.exists():
         cli.log.error('Keyboard default %s does not exist!', keymap_path_default)
         exit(1)
-    if os.path.exists(keymap_path):
-        cli.log.error('Keymap %s already exists!', keymap_path)
+
+    if keymap_path_new.exists():
+        cli.log.error('Keymap %s already exists!', keymap_path_new)
         exit(1)
 
     # create user directory with default keymap files
-    shutil.copytree(keymap_path_default, keymap_path, symlinks=True)
+    shutil.copytree(str(keymap_path_default), str(keymap_path_new), symlinks=True)
 
     # end message to user
-    cli.log.info("%s keymap directory created in: %s", keymap, keymap_path)
-    cli.log.info("Compile a firmware with your new keymap by typing: \n" + "qmk compile -kb %s -km %s", keyboard, keymap)
+    cli.log.info("%s keymap directory created in: %s", keymap, keymap_path_new)
+    cli.log.info("Compile a firmware with your new keymap by typing: \n\n\tqmk compile -kb %s -km %s\n", keyboard, keymap)