From fb216c21fba9f61fc5923f9069a9cf3417da8d0e Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 28 Aug 2023 21:32:05 +1000
Subject: [PATCH 01/84] Userspace support.

---
 Makefile                           |  25 ++++++-
 builddefs/build_json.mk            |  19 ++++++
 builddefs/build_keyboard.mk        |  81 +++++++++++++++--------
 builddefs/build_layout.mk          |   4 ++
 builddefs/common_rules.mk          |  11 ++-
 lib/python/qmk/cli/mass_compile.py |   9 ++-
 lib/python/qmk/commands.py         |   5 +-
 lib/python/qmk/constants.py        |   3 +
 lib/python/qmk/keyboard.py         |   4 +-
 lib/python/qmk/keymap.py           | 103 +++++++++++++++++------------
 lib/python/qmk/path.py             |  23 +++++--
 11 files changed, 207 insertions(+), 80 deletions(-)

diff --git a/Makefile b/Makefile
index 9f2e4636a4b4..c6c2fe4956f4 100644
--- a/Makefile
+++ b/Makefile
@@ -191,9 +191,20 @@ define PARSE_KEYBOARD
     KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/keyboards/$$(KEYBOARD_FOLDER_PATH_4)/keymaps/*/.)))
     KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/keyboards/$$(KEYBOARD_FOLDER_PATH_5)/keymaps/*/.)))
 
+    ifneq ($(QMK_USERSPACE),)
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_1)/keymaps/*/.)))
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_2)/keymaps/*/.)))
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_3)/keymaps/*/.)))
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_4)/keymaps/*/.)))
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_5)/keymaps/*/.)))
+    endif
+
     KEYBOARD_LAYOUTS := $(shell $(QMK_BIN) list-layouts --keyboard $1)
     LAYOUT_KEYMAPS :=
     $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/layouts/*/$$(LAYOUT)/*/.)))))
+    ifneq ($(QMK_USERSPACE),)
+        $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/layouts/$$(LAYOUT)/*/.)))))
+    endif
 
     KEYMAPS := $$(sort $$(KEYMAPS) $$(LAYOUT_KEYMAPS))
 
@@ -423,8 +434,18 @@ clean:
 	rm -rf $(BUILD_DIR)
 	echo 'done.'
 
-.PHONY: distclean
-distclean: clean
+.PHONY: distclean distclean_qmk
+distclean: distclean_qmk
+distclean_qmk: clean
 	echo -n 'Deleting *.bin, *.hex, and *.uf2 ... '
 	rm -f *.bin *.hex *.uf2
 	echo 'done.'
+
+ifneq ($(QMK_USERSPACE),)
+.PHONY: distclean_userspace
+distclean: distclean_userspace
+distclean_userspace: clean
+	echo -n 'Deleting user overlay *.bin, *.hex, and *.uf2 ... '
+	rm -f $(QMK_USERSPACE)/*.bin $(QMK_USERSPACE)/*.hex $(QMK_USERSPACE)/*.uf2
+	echo 'done.'
+endif
diff --git a/builddefs/build_json.mk b/builddefs/build_json.mk
index 0c034eb2aea0..67082340ec22 100644
--- a/builddefs/build_json.mk
+++ b/builddefs/build_json.mk
@@ -15,3 +15,22 @@ else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
     KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
     KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
 endif
+
+ifneq ($(QMK_USERSPACE),)
+    ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)
+    else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)
+    else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)
+    else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)
+    else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)
+    endif
+endif
diff --git a/builddefs/build_keyboard.mk b/builddefs/build_keyboard.mk
index e93ab97cc134..998cdd66cda9 100644
--- a/builddefs/build_keyboard.mk
+++ b/builddefs/build_keyboard.mk
@@ -129,32 +129,57 @@ include $(BUILDDEFS_PATH)/build_json.mk
 # Pull in keymap level rules.mk
 ifeq ("$(wildcard $(KEYMAP_PATH))", "")
     # Look through the possible keymap folders until we find a matching keymap.c
-    ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
-        -include $(MAIN_KEYMAP_PATH_1)/rules.mk
-        KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
-        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
-    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
-        -include $(MAIN_KEYMAP_PATH_2)/rules.mk
-        KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
-        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
-    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
-        -include $(MAIN_KEYMAP_PATH_3)/rules.mk
-        KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
-        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
-    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
-        -include $(MAIN_KEYMAP_PATH_4)/rules.mk
-        KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
-        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
-    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
-        -include $(MAIN_KEYMAP_PATH_5)/rules.mk
-        KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
-        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
-    else ifneq ($(LAYOUTS),)
-        # If we haven't found a keymap yet fall back to community layouts
-        include $(BUILDDEFS_PATH)/build_layout.mk
-    else
-        $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap)
-        # this state should never be reached
+    ifneq ($(QMK_USERSPACE),)
+        ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)
+        else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)
+        else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)
+        else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)
+        else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)
+        endif
+    endif
+    ifeq ($(KEYMAP_PATH),)
+        ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_1)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+        else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_2)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+        else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_3)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+        else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_4)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+        else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_5)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+        else ifneq ($(LAYOUTS),)
+            # If we haven't found a keymap yet fall back to community layouts
+            include $(BUILDDEFS_PATH)/build_layout.mk
+        else
+            $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap)
+            # this state should never be reached
+        endif
     endif
 endif
 
@@ -359,6 +384,10 @@ ifeq ("$(USER_NAME)","")
 endif
 USER_PATH := users/$(USER_NAME)
 
+ifneq ($(QMK_USERSPACE),)
+    USER_PATH := $(QMK_USERSPACE)/$(USER_PATH)
+endif
+
 # Pull in user level rules.mk
 -include $(USER_PATH)/rules.mk
 ifneq ("$(wildcard $(USER_PATH)/config.h)","")
diff --git a/builddefs/build_layout.mk b/builddefs/build_layout.mk
index 6166bd847c59..9ff99cc2218e 100644
--- a/builddefs/build_layout.mk
+++ b/builddefs/build_layout.mk
@@ -1,6 +1,10 @@
 LAYOUTS_PATH := layouts
 LAYOUTS_REPOS := $(patsubst %/,%,$(sort $(dir $(wildcard $(LAYOUTS_PATH)/*/))))
 
+ifneq ($(QMK_USERSPACE),)
+    LAYOUTS_REPOS += $(patsubst %/,%,$(QMK_USERSPACE)/$(LAYOUTS_PATH))
+endif
+
 define SEARCH_LAYOUTS_REPO
     LAYOUT_KEYMAP_PATH := $$(LAYOUTS_REPO)/$$(LAYOUT)/$$(KEYMAP)
     LAYOUT_KEYMAP_JSON := $$(LAYOUT_KEYMAP_PATH)/keymap.json
diff --git a/builddefs/common_rules.mk b/builddefs/common_rules.mk
index 5d635307edb9..ae17ad127e96 100644
--- a/builddefs/common_rules.mk
+++ b/builddefs/common_rules.mk
@@ -188,7 +188,7 @@ DFU_SUFFIX_ARGS ?=
 elf: $(BUILD_DIR)/$(TARGET).elf
 hex: $(BUILD_DIR)/$(TARGET).hex
 uf2: $(BUILD_DIR)/$(TARGET).uf2
-cpfirmware: $(FIRMWARE_FORMAT)
+cpfirmware_qmk: $(FIRMWARE_FORMAT)
 	$(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to qmk_firmware folder" | $(AWK_CMD)
 	$(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK)
 eep: $(BUILD_DIR)/$(TARGET).eep
@@ -197,6 +197,15 @@ sym: $(BUILD_DIR)/$(TARGET).sym
 LIBNAME=lib$(TARGET).a
 lib: $(LIBNAME)
 
+cpfirmware: cpfirmware_qmk
+
+ifneq ($(QMK_USERSPACE),)
+cpfirmware: cpfirmware_userspace
+cpfirmware_userspace: cpfirmware_qmk
+	$(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to user overlay folder" | $(AWK_CMD)
+	$(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(QMK_USERSPACE)/$(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK)
+endif
+
 # Display size of file, modifying the output so people don't mistakenly grab the hex output
 BINARY_SIZE = $(SIZE) --target=$(FORMAT) $(BUILD_DIR)/$(TARGET).hex | $(SED) -e 's/\.build\/.*$$/$(TARGET).$(FIRMWARE_FORMAT)/g'
 
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index ddd946a32bc6..9dad5420e9d6 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -7,7 +7,7 @@
 from subprocess import DEVNULL
 from milc import cli
 
-from qmk.constants import QMK_FIRMWARE
+from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE
 from qmk.commands import _find_make, get_make_parallel_args
 from qmk.keyboard import resolve_keyboard
 from qmk.search import search_keymap_targets
@@ -49,6 +49,11 @@ def mass_compile(cli):
 
     builddir.mkdir(parents=True, exist_ok=True)
     with open(makefile, "w") as f:
+
+        userspace_suffix = ''
+        if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve():
+            userspace_suffix = f'QMK_USERSPACE={Path(QMK_USERSPACE).resolve()}'
+
         for target in sorted(targets):
             keyboard_name = target[0]
             keymap_name = target[1]
@@ -62,7 +67,7 @@ def mass_compile(cli):
 {keyboard_safe}_{keymap_name}_binary:
 	@rm -f "{build_log}" || true
 	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(cli.args.env)} \\
+	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {userspace_suffix} {' '.join(cli.args.env)} \\
 		>>"{build_log}" 2>&1 \\
 		|| cp "{build_log}" "{failed_log}"
 	@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 660b2ff72e60..4b1fea6bbb8c 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -9,7 +9,7 @@
 from milc import cli
 import jsonschema
 
-from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX
+from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX, QMK_FIRMWARE, QMK_USERSPACE
 from qmk.json_schema import json_load, validate
 
 
@@ -51,6 +51,9 @@ def create_make_target(target, dry_run=False, parallel=1, **env_vars):
     for key, value in env_vars.items():
         env.append(f'{key}={value}')
 
+    if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve():
+        env.append(f'QMK_USERSPACE={Path(QMK_USERSPACE).resolve()}')
+
     if cli.config.general.verbose:
         env.append('VERBOSE=true')
 
diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index 97bd84aa2344..6db38c3049fa 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -4,8 +4,11 @@
 from datetime import date
 from pathlib import Path
 
+from milc import cli
+
 # The root of the qmk_firmware tree.
 QMK_FIRMWARE = Path.cwd()
+QMK_USERSPACE = environ.get('QMK_USERSPACE') or cli.config.user.overlay_dir or QMK_FIRMWARE
 
 # Upstream repo url
 QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 235b62640c20..ff3e9f39cae7 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -36,7 +36,9 @@
 def find_keyboard_from_dir():
     """Returns a keyboard name based on the user's current directory.
     """
-    relative_cwd = qmk.path.under_qmk_firmware()
+    relative_cwd = qmk.path.under_qmk_userspace()
+    if not relative_cwd:
+        relative_cwd = qmk.path.under_qmk_firmware()
 
     if relative_cwd and len(relative_cwd.parts) > 1 and relative_cwd.parts[0] == 'keyboards':
         # Attempt to extract the keyboard name from the current directory
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index 11e8d39dadf5..bac919a773f4 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -12,6 +12,7 @@
 from pygments import lex
 
 import qmk.path
+from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE
 from qmk.keyboard import find_keyboard_from_dir, keyboard_folder
 from qmk.errors import CppError
 from qmk.info import info_json
@@ -195,29 +196,38 @@ def find_keymap_from_dir():
     """Returns `(keymap_name, source)` for the directory we're currently in.
 
     """
-    relative_cwd = qmk.path.under_qmk_firmware()
+    def _impl_find_keymap_from_dir(relative_cwd):
+        if relative_cwd and len(relative_cwd.parts) > 1:
+            # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name.
+            if relative_cwd.parts[0] == 'keyboards' and 'keymaps' in relative_cwd.parts:
+                current_path = Path('/'.join(relative_cwd.parts[1:]))  # Strip 'keyboards' from the front
 
-    if relative_cwd and len(relative_cwd.parts) > 1:
-        # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name.
-        if relative_cwd.parts[0] == 'keyboards' and 'keymaps' in relative_cwd.parts:
-            current_path = Path('/'.join(relative_cwd.parts[1:]))  # Strip 'keyboards' from the front
+                if 'keymaps' in current_path.parts and current_path.name != 'keymaps':
+                    while current_path.parent.name != 'keymaps':
+                        current_path = current_path.parent
 
-            if 'keymaps' in current_path.parts and current_path.name != 'keymaps':
-                while current_path.parent.name != 'keymaps':
-                    current_path = current_path.parent
+                    return current_path.name, 'keymap_directory'
 
-                return current_path.name, 'keymap_directory'
+            # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in
+            elif relative_cwd.parts[0] == 'layouts' and is_keymap_dir(relative_cwd):
+                return relative_cwd.name, 'layouts_directory'
 
-        # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in
-        elif relative_cwd.parts[0] == 'layouts' and is_keymap_dir(relative_cwd):
-            return relative_cwd.name, 'layouts_directory'
+            # If we're in `qmk_firmware/users` guess the name from the userspace they're in
+            elif relative_cwd.parts[0] == 'users':
+                # Guess the keymap name based on which userspace they're in
+                return relative_cwd.parts[1], 'users_directory'
+        return None, None
 
-        # If we're in `qmk_firmware/users` guess the name from the userspace they're in
-        elif relative_cwd.parts[0] == 'users':
-            # Guess the keymap name based on which userspace they're in
-            return relative_cwd.parts[1], 'users_directory'
+    if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve():
+        name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_userspace())
+        if name and source:
+            return name, source
 
-    return None, None
+    name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_firmware())
+    if name and source:
+        return name, source
+
+    return (None, None)
 
 
 def keymap_completer(prefix, action, parser, parsed_args):
@@ -418,21 +428,22 @@ def locate_keymap(keyboard, keymap):
         raise KeyError('Invalid keyboard: ' + repr(keyboard))
 
     # Check the keyboard folder first, last match wins
-    checked_dirs = ''
     keymap_path = ''
 
-    for dir in keyboard_folder(keyboard).split('/'):
-        if checked_dirs:
-            checked_dirs = '/'.join((checked_dirs, dir))
-        else:
-            checked_dirs = dir
+    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE]:
+        checked_dirs = ''
+        for dir in keyboard_folder(keyboard).split('/'):
+            if checked_dirs:
+                checked_dirs = '/'.join((checked_dirs, dir))
+            else:
+                checked_dirs = dir
 
-        keymap_dir = Path('keyboards') / checked_dirs / 'keymaps'
+            keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps'
 
-        if (keymap_dir / keymap / 'keymap.c').exists():
-            keymap_path = keymap_dir / keymap / 'keymap.c'
-        if (keymap_dir / keymap / 'keymap.json').exists():
-            keymap_path = keymap_dir / keymap / 'keymap.json'
+            if (keymap_dir / keymap / 'keymap.c').exists():
+                keymap_path = keymap_dir / keymap / 'keymap.c'
+            if (keymap_dir / keymap / 'keymap.json').exists():
+                keymap_path = keymap_dir / keymap / 'keymap.json'
 
     if keymap_path:
         return keymap_path
@@ -440,7 +451,11 @@ def locate_keymap(keyboard, keymap):
     # Check community layouts as a fallback
     info = info_json(keyboard)
 
-    for community_parent in Path('layouts').glob('*/'):
+    community_parents = list(Path('layouts').glob('*/'))
+    if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve() and (Path(QMK_USERSPACE) / "layouts").exists():
+        community_parents.append(Path(QMK_USERSPACE) / "layouts")
+
+    for community_parent in community_parents:
         for layout in info.get("community_layouts", []):
             community_layout = community_parent / layout / keymap
             if community_layout.exists():
@@ -474,26 +489,30 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     """
     names = set()
 
-    keyboards_dir = Path('keyboards')
-    kb_path = keyboards_dir / keyboard
-
     # walk up the directory tree until keyboards_dir
     # and collect all directories' name with keymap.c file in it
-    while kb_path != keyboards_dir:
-        keymaps_dir = kb_path / "keymaps"
-
-        if keymaps_dir.is_dir():
-            for keymap in keymaps_dir.iterdir():
-                if is_keymap_dir(keymap, c, json, additional_files):
-                    keymap = keymap if fullpath else keymap.name
-                    names.add(keymap)
+    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE]:
+        keyboards_dir = search_dir / Path('keyboards')
+        kb_path = keyboards_dir / keyboard
+
+        while kb_path != keyboards_dir:
+            keymaps_dir = kb_path / "keymaps"
+            if keymaps_dir.is_dir():
+                for keymap in keymaps_dir.iterdir():
+                    if is_keymap_dir(keymap, c, json, additional_files):
+                        keymap = keymap if fullpath else keymap.name
+                        names.add(keymap)
 
-        kb_path = kb_path.parent
+            kb_path = kb_path.parent
 
     # Check community layouts as a fallback
     info = info_json(keyboard)
 
-    for community_parent in Path('layouts').glob('*/'):
+    community_parents = list(Path('layouts').glob('*/'))
+    if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve() and (Path(QMK_USERSPACE) / "layouts").exists():
+        community_parents.append(Path(QMK_USERSPACE) / "layouts")
+
+    for community_parent in community_parents:
         for layout in info.get("community_layouts", []):
             cl_path = community_parent / layout
             if cl_path.is_dir():
diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py
index 9d248451b86f..3a069e2c15dd 100644
--- a/lib/python/qmk/path.py
+++ b/lib/python/qmk/path.py
@@ -5,7 +5,7 @@
 import argparse
 from pathlib import Path
 
-from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE
+from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE, QMK_USERSPACE
 from qmk.errors import NoSuchKeyboardError
 
 
@@ -30,6 +30,17 @@ def under_qmk_firmware():
         return None
 
 
+def under_qmk_userspace():
+    """Returns a Path object representing the relative path under $QMK_USERSPACE, qmk root, or None.
+    """
+    cwd = Path(os.environ['ORIG_CWD'])
+
+    try:
+        return cwd.relative_to(QMK_USERSPACE)
+    except ValueError:
+        return None
+
+
 def keyboard(keyboard_name):
     """Returns the path to a keyboard's directory relative to the qmk root.
     """
@@ -47,11 +58,13 @@ def keymaps(keyboard_name):
     keyboard_folder = keyboard(keyboard_name)
     found_dirs = []
 
-    for _ in range(MAX_KEYBOARD_SUBFOLDERS):
-        if (keyboard_folder / 'keymaps').exists():
-            found_dirs.append((keyboard_folder / 'keymaps').resolve())
+    for root_dir in [QMK_USERSPACE, QMK_FIRMWARE]:
+        this_keyboard_folder = root_dir / keyboard_folder
+        for _ in range(MAX_KEYBOARD_SUBFOLDERS):
+            if (this_keyboard_folder / 'keymaps').exists():
+                found_dirs.append((this_keyboard_folder / 'keymaps').resolve())
 
-        keyboard_folder = keyboard_folder.parent
+            this_keyboard_folder = this_keyboard_folder.parent
 
     if len(found_dirs) > 0:
         return found_dirs

From b441da6a853ab901cd1445c1a07d5da9f8c29bd3 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Thu, 31 Aug 2023 20:06:16 +1000
Subject: [PATCH 02/84] Auto-detect userspace directory.

---
 lib/python/qmk/constants.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index 6db38c3049fa..58d00faa528a 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -8,7 +8,22 @@
 
 # The root of the qmk_firmware tree.
 QMK_FIRMWARE = Path.cwd()
-QMK_USERSPACE = environ.get('QMK_USERSPACE') or cli.config.user.overlay_dir or QMK_FIRMWARE
+
+
+# Helper to detect userspace
+def _detect_qmk_userspace():
+    # If we're already in a directory with a Makefile and a keyboards or layouts directory, interpret it as userspace
+    current_dir = Path(environ['ORIG_CWD'])
+    while len(current_dir.parts) > 1:
+        if (current_dir / 'Makefile').is_file() and ((current_dir / 'keyboards').is_dir() or (current_dir / 'layouts').is_dir()):
+            return current_dir
+        current_dir = current_dir.parent
+    # Otherwise, use the environment variable or the configured default
+    return environ.get('QMK_USERSPACE') or cli.config.user.overlay_dir or QMK_FIRMWARE
+
+
+# The detected userspace tree
+QMK_USERSPACE = _detect_qmk_userspace()
 
 # Upstream repo url
 QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'

From 5477193489064562b9e3e38cca4164f510c4a24b Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 2 Sep 2023 20:50:49 +1000
Subject: [PATCH 03/84] `qmk new-keymap` now locates the new keymap inside
 userspace if set.

---
 lib/python/qmk/cli/mass_compile.py |  4 ++--
 lib/python/qmk/cli/new/keymap.py   |  5 +++++
 lib/python/qmk/commands.py         |  4 ++--
 lib/python/qmk/constants.py        |  3 +++
 lib/python/qmk/keymap.py           | 10 +++++-----
 5 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 9dad5420e9d6..b978a1ef7ced 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -7,7 +7,7 @@
 from subprocess import DEVNULL
 from milc import cli
 
-from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE
+from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.commands import _find_make, get_make_parallel_args
 from qmk.keyboard import resolve_keyboard
 from qmk.search import search_keymap_targets
@@ -51,7 +51,7 @@ def mass_compile(cli):
     with open(makefile, "w") as f:
 
         userspace_suffix = ''
-        if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve():
+        if HAS_QMK_USERSPACE:
             userspace_suffix = f'QMK_USERSPACE={Path(QMK_USERSPACE).resolve()}'
 
         for target in sorted(targets):
diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index 9b0ac221a4ae..f7ffcab0e992 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -1,10 +1,12 @@
 """This script automates the copying of the default keymap into your own keymap.
 """
 import shutil
+from pathlib import Path
 
 from milc import cli
 from milc.questions import question
 
+from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE, QMK_FIRMWARE
 from qmk.path import is_keyboard, keymaps, keymap
 from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
@@ -54,6 +56,9 @@ def new_keymap(cli):
     keymap_path_default = keymap(kb_name, 'default')
     keymap_path_new = keymaps_dirs[0] / user_name
 
+    if HAS_QMK_USERSPACE:
+        keymap_path_new = Path(QMK_USERSPACE) / keymaps_dirs[0].relative_to(QMK_FIRMWARE) / user_name
+
     if not keymap_path_default.exists():
         cli.log.error(f'Default keymap {{fg_cyan}}{keymap_path_default}{{fg_reset}} does not exist!')
         return False
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 4b1fea6bbb8c..ad73fc58d16e 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -9,7 +9,7 @@
 from milc import cli
 import jsonschema
 
-from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX, QMK_FIRMWARE, QMK_USERSPACE
+from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX, QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.json_schema import json_load, validate
 
 
@@ -51,7 +51,7 @@ def create_make_target(target, dry_run=False, parallel=1, **env_vars):
     for key, value in env_vars.items():
         env.append(f'{key}={value}')
 
-    if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve():
+    if HAS_QMK_USERSPACE:
         env.append(f'QMK_USERSPACE={Path(QMK_USERSPACE).resolve()}')
 
     if cli.config.general.verbose:
diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index 58d00faa528a..b9555948e9fb 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -25,6 +25,9 @@ def _detect_qmk_userspace():
 # The detected userspace tree
 QMK_USERSPACE = _detect_qmk_userspace()
 
+# Whether or not we have a separate userspace directory
+HAS_QMK_USERSPACE = True if Path(QMK_USERSPACE).resolve() != Path(QMK_FIRMWARE).resolve() else False
+
 # Upstream repo url
 QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'
 
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index bac919a773f4..f3c620ed8ef8 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -12,7 +12,7 @@
 from pygments import lex
 
 import qmk.path
-from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE
+from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.keyboard import find_keyboard_from_dir, keyboard_folder
 from qmk.errors import CppError
 from qmk.info import info_json
@@ -218,7 +218,7 @@ def _impl_find_keymap_from_dir(relative_cwd):
                 return relative_cwd.parts[1], 'users_directory'
         return None, None
 
-    if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve():
+    if HAS_QMK_USERSPACE:
         name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_userspace())
         if name and source:
             return name, source
@@ -430,7 +430,7 @@ def locate_keymap(keyboard, keymap):
     # Check the keyboard folder first, last match wins
     keymap_path = ''
 
-    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE]:
+    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]:
         checked_dirs = ''
         for dir in keyboard_folder(keyboard).split('/'):
             if checked_dirs:
@@ -452,7 +452,7 @@ def locate_keymap(keyboard, keymap):
     info = info_json(keyboard)
 
     community_parents = list(Path('layouts').glob('*/'))
-    if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve() and (Path(QMK_USERSPACE) / "layouts").exists():
+    if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists():
         community_parents.append(Path(QMK_USERSPACE) / "layouts")
 
     for community_parent in community_parents:
@@ -509,7 +509,7 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     info = info_json(keyboard)
 
     community_parents = list(Path('layouts').glob('*/'))
-    if Path(QMK_FIRMWARE).resolve() != Path(QMK_USERSPACE).resolve() and (Path(QMK_USERSPACE) / "layouts").exists():
+    if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists():
         community_parents.append(Path(QMK_USERSPACE) / "layouts")
 
     for community_parent in community_parents:

From da39824043ea82cd946af4064e681c17352d3137 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 2 Sep 2023 21:49:31 +1000
Subject: [PATCH 04/84] Don't accidentally resolve qmk_firmware as userspace if
 it's the current working directory.

---
 lib/python/qmk/constants.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index b9555948e9fb..6d6c3dedda03 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -16,7 +16,8 @@ def _detect_qmk_userspace():
     current_dir = Path(environ['ORIG_CWD'])
     while len(current_dir.parts) > 1:
         if (current_dir / 'Makefile').is_file() and ((current_dir / 'keyboards').is_dir() or (current_dir / 'layouts').is_dir()):
-            return current_dir
+            if current_dir.resolve() != Path(QMK_FIRMWARE).resolve():
+                return current_dir
         current_dir = current_dir.parent
     # Otherwise, use the environment variable or the configured default
     return environ.get('QMK_USERSPACE') or cli.config.user.overlay_dir or QMK_FIRMWARE

From 926ad823673384678162c23915309cb2522d0b70 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 3 Sep 2023 17:28:39 +1000
Subject: [PATCH 05/84] Userspace workflow.

---
 .github/workflows/qmk_userspace_build.yml | 105 ++++++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 .github/workflows/qmk_userspace_build.yml

diff --git a/.github/workflows/qmk_userspace_build.yml b/.github/workflows/qmk_userspace_build.yml
new file mode 100644
index 000000000000..735e898c7ff8
--- /dev/null
+++ b/.github/workflows/qmk_userspace_build.yml
@@ -0,0 +1,105 @@
+name: Build Binaries
+
+on:
+  workflow_call:
+    inputs:
+      qmk_repo:
+        description: 'qmk_firmware repo to build against'
+        default: 'qmk/qmk_firmware'
+        required: false
+        type: string
+      qmk_ref:
+        description: 'qmk_firmware branch to build against'
+        default: 'master'
+        required: false
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    container: qmkfm/base_container
+
+    steps:
+    - name: Checkout Userspace
+      uses: actions/checkout@v3
+      with:
+        path: userspace
+        submodules: recursive
+
+    # This doesn't use actions/checkout as we want to allow qmk_firmware as a submodule
+    - name: Checkout QMK Firmware
+      run: |
+        if [ ! -d qmk_firmware ]; then
+          git clone -b ${{ inputs.qmk_ref || 'master' }} \
+            --depth=1000 --recurse-submodules --shallow-submodules \
+            https://github.com/${{ inputs.qmk_repo || 'qmk/qmk_firmware' }}.git
+        fi
+
+    ########################################################
+    ## Delete this section once userspace exists upstream ##
+    ########################################################
+    - name: Patch QMK Firmware with userspace support
+      run: |
+        cd qmk_firmware
+        git config --global user.email "nonexistent@email-address.invalid"
+        git config --global user.name "QMK GitHub Actions User"
+        git remote add qmk https://github.com/qmk/qmk_firmware.git
+        git fetch qmk qmk-userspace
+        git merge --no-commit --squash qmk/qmk-userspace
+        cd ..
+
+    - name: Install QMK CLI
+      run: |
+        python3 -m pip install --upgrade qmk
+        python3 -m pip install -r qmk_firmware/requirements.txt
+
+    - name: Configure QMK CLI
+      run: |
+        qmk config mass_compile.parallel=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
+        qmk config user.qmk_home=$GITHUB_WORKSPACE/qmk_firmware
+        qmk config user.overlay_dir=$GITHUB_WORKSPACE/userspace
+
+    - name: Build
+      run: |
+        qmk mass-compile -e DUMP_CI_METADATA=yes -km ${{ github.repository_owner }} || touch .failed
+        # Generate the step summary markdown
+        ./qmk_firmware/util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
+        # Truncate to a maximum of 1MB to deal with GitHub workflow limit
+        truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
+        # Exit with failure if the compilation stage failed
+        [ ! -f .failed ] || exit 1
+
+    - name: Upload binaries
+      uses: actions/upload-artifact@v3
+      if: always()
+      continue-on-error: true
+      with:
+        name: Firmware
+        path: |
+          userspace/*.bin
+          userspace/*.hex
+          userspace/*.uf2
+
+  publish:
+    name: Publish
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/download-artifact@v3
+
+    - uses: marvinpinto/action-automatic-releases@latest
+      if: always()
+      with:
+        repo_token: "${{ github.token }}"
+        automatic_release_tag: "latest"
+        title: "Latest Firmware"
+        prerelease: false
+        files: |
+          **/*.hex
+          **/*.bin
+          **/*.uf2

From a750b85aa17aeb645d55a13db09a750ca42b550e Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 3 Sep 2023 21:17:01 +1000
Subject: [PATCH 06/84] Workflow update.

---
 .github/workflows/qmk_userspace_build.yml | 29 ++++++++++++++---------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/qmk_userspace_build.yml b/.github/workflows/qmk_userspace_build.yml
index 735e898c7ff8..81b07a77924c 100644
--- a/.github/workflows/qmk_userspace_build.yml
+++ b/.github/workflows/qmk_userspace_build.yml
@@ -30,21 +30,28 @@ jobs:
         path: userspace
         submodules: recursive
 
-    # This doesn't use actions/checkout as we want to allow qmk_firmware as a submodule
+    - name: Check qmk_firmware exists
+      id: check_files
+      uses: andstor/file-existence-action@v2
+      with:
+        files: userspace/qmk_firmware
+
     - name: Checkout QMK Firmware
-      run: |
-        if [ ! -d qmk_firmware ]; then
-          git clone -b ${{ inputs.qmk_ref || 'master' }} \
-            --depth=1000 --recurse-submodules --shallow-submodules \
-            https://github.com/${{ inputs.qmk_repo || 'qmk/qmk_firmware' }}.git
-        fi
+      uses: actions/checkout@v3
+      if: steps.check_files.outputs.files_exists == 'false'
+      with:
+        path: userspace/qmk_firmware
+        repository: ${{ inputs.qmk_repo || 'qmk/qmk_firmware' }}
+        ref: ${{ inputs.qmk_ref || 'master' }}
+        submodules: recursive
+        fetch-depth: 1000 # This line should be removed once userspace support exists in QMK Firmware
 
     ########################################################
     ## Delete this section once userspace exists upstream ##
     ########################################################
     - name: Patch QMK Firmware with userspace support
       run: |
-        cd qmk_firmware
+        cd userspace/qmk_firmware
         git config --global user.email "nonexistent@email-address.invalid"
         git config --global user.name "QMK GitHub Actions User"
         git remote add qmk https://github.com/qmk/qmk_firmware.git
@@ -55,19 +62,19 @@ jobs:
     - name: Install QMK CLI
       run: |
         python3 -m pip install --upgrade qmk
-        python3 -m pip install -r qmk_firmware/requirements.txt
+        python3 -m pip install -r userspace/qmk_firmware/requirements.txt
 
     - name: Configure QMK CLI
       run: |
         qmk config mass_compile.parallel=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
-        qmk config user.qmk_home=$GITHUB_WORKSPACE/qmk_firmware
+        qmk config user.qmk_home=$GITHUB_WORKSPACE/userspace/qmk_firmware
         qmk config user.overlay_dir=$GITHUB_WORKSPACE/userspace
 
     - name: Build
       run: |
         qmk mass-compile -e DUMP_CI_METADATA=yes -km ${{ github.repository_owner }} || touch .failed
         # Generate the step summary markdown
-        ./qmk_firmware/util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
+        ./userspace/qmk_firmware/util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
         # Truncate to a maximum of 1MB to deal with GitHub workflow limit
         truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
         # Exit with failure if the compilation stage failed

From c4b133ca792e34d40f92f9774b6fdc7a3e4e23a0 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 3 Sep 2023 21:36:16 +1000
Subject: [PATCH 07/84] Workflow paths.

---
 .github/workflows/qmk_userspace_build.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/qmk_userspace_build.yml b/.github/workflows/qmk_userspace_build.yml
index 81b07a77924c..6689b9ce2b80 100644
--- a/.github/workflows/qmk_userspace_build.yml
+++ b/.github/workflows/qmk_userspace_build.yml
@@ -27,20 +27,19 @@ jobs:
     - name: Checkout Userspace
       uses: actions/checkout@v3
       with:
-        path: userspace
         submodules: recursive
 
     - name: Check qmk_firmware exists
       id: check_files
       uses: andstor/file-existence-action@v2
       with:
-        files: userspace/qmk_firmware
+        files: qmk_firmware
 
     - name: Checkout QMK Firmware
       uses: actions/checkout@v3
       if: steps.check_files.outputs.files_exists == 'false'
       with:
-        path: userspace/qmk_firmware
+        path: qmk_firmware
         repository: ${{ inputs.qmk_repo || 'qmk/qmk_firmware' }}
         ref: ${{ inputs.qmk_ref || 'master' }}
         submodules: recursive
@@ -51,7 +50,7 @@ jobs:
     ########################################################
     - name: Patch QMK Firmware with userspace support
       run: |
-        cd userspace/qmk_firmware
+        cd qmk_firmware
         git config --global user.email "nonexistent@email-address.invalid"
         git config --global user.name "QMK GitHub Actions User"
         git remote add qmk https://github.com/qmk/qmk_firmware.git
@@ -62,19 +61,19 @@ jobs:
     - name: Install QMK CLI
       run: |
         python3 -m pip install --upgrade qmk
-        python3 -m pip install -r userspace/qmk_firmware/requirements.txt
+        python3 -m pip install -r qmk_firmware/requirements.txt
 
     - name: Configure QMK CLI
       run: |
         qmk config mass_compile.parallel=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
-        qmk config user.qmk_home=$GITHUB_WORKSPACE/userspace/qmk_firmware
-        qmk config user.overlay_dir=$GITHUB_WORKSPACE/userspace
+        qmk config user.qmk_home=$GITHUB_WORKSPACE/qmk_firmware
+        qmk config user.overlay_dir=$GITHUB_WORKSPACE
 
     - name: Build
       run: |
         qmk mass-compile -e DUMP_CI_METADATA=yes -km ${{ github.repository_owner }} || touch .failed
         # Generate the step summary markdown
-        ./userspace/qmk_firmware/util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
+        ./qmk_firmware/util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
         # Truncate to a maximum of 1MB to deal with GitHub workflow limit
         truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
         # Exit with failure if the compilation stage failed

From d1562c534330ec30170a7f78747a2aa095503b01 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 3 Sep 2023 21:40:57 +1000
Subject: [PATCH 08/84] Swapped GHA for release

---
 .github/workflows/qmk_userspace_build.yml | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/qmk_userspace_build.yml b/.github/workflows/qmk_userspace_build.yml
index 6689b9ce2b80..210bfd4fa636 100644
--- a/.github/workflows/qmk_userspace_build.yml
+++ b/.github/workflows/qmk_userspace_build.yml
@@ -86,9 +86,9 @@ jobs:
       with:
         name: Firmware
         path: |
-          userspace/*.bin
-          userspace/*.hex
-          userspace/*.uf2
+          *.bin
+          *.hex
+          *.uf2
 
   publish:
     name: Publish
@@ -96,14 +96,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/download-artifact@v3
+    - name: Download binaries
+      uses: actions/download-artifact@v3
 
-    - uses: marvinpinto/action-automatic-releases@latest
+    - name: Generate Release
+      uses: softprops/action-gh-release@v1
       if: always()
       with:
-        repo_token: "${{ github.token }}"
-        automatic_release_tag: "latest"
-        title: "Latest Firmware"
+        token: "${{ github.token }}"
+        name: Latest Firmware
+        tag_name: latest
+        fail_on_unmatched_files: false
+        draft: false
         prerelease: false
         files: |
           **/*.hex

From 3716172f202b9c0a720dd82f1bd6f4a48e34f5c7 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 3 Sep 2023 22:03:37 +1000
Subject: [PATCH 09/84] Review comments

---
 .github/workflows/qmk_userspace_build.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/qmk_userspace_build.yml b/.github/workflows/qmk_userspace_build.yml
index 210bfd4fa636..12ea88c97589 100644
--- a/.github/workflows/qmk_userspace_build.yml
+++ b/.github/workflows/qmk_userspace_build.yml
@@ -81,7 +81,7 @@ jobs:
 
     - name: Upload binaries
       uses: actions/upload-artifact@v3
-      if: always()
+      if: always() && !cancelled()
       continue-on-error: true
       with:
         name: Firmware
@@ -97,11 +97,12 @@ jobs:
 
     steps:
     - name: Download binaries
+      if: always() && !cancelled()
       uses: actions/download-artifact@v3
 
     - name: Generate Release
       uses: softprops/action-gh-release@v1
-      if: always()
+      if: always() && !cancelled()
       with:
         token: "${{ github.token }}"
         name: Latest Firmware

From 7521a1181bf831397d61f951dc29681a6648b98f Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 4 Sep 2023 22:00:10 +1000
Subject: [PATCH 10/84] Use `qmk/.github` for workflow definition.

---
 .github/workflows/qmk_userspace_build.yml | 116 ----------------------
 1 file changed, 116 deletions(-)
 delete mode 100644 .github/workflows/qmk_userspace_build.yml

diff --git a/.github/workflows/qmk_userspace_build.yml b/.github/workflows/qmk_userspace_build.yml
deleted file mode 100644
index 12ea88c97589..000000000000
--- a/.github/workflows/qmk_userspace_build.yml
+++ /dev/null
@@ -1,116 +0,0 @@
-name: Build Binaries
-
-on:
-  workflow_call:
-    inputs:
-      qmk_repo:
-        description: 'qmk_firmware repo to build against'
-        default: 'qmk/qmk_firmware'
-        required: false
-        type: string
-      qmk_ref:
-        description: 'qmk_firmware branch to build against'
-        default: 'master'
-        required: false
-        type: string
-
-permissions:
-  contents: write
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    container: qmkfm/base_container
-
-    steps:
-    - name: Checkout Userspace
-      uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - name: Check qmk_firmware exists
-      id: check_files
-      uses: andstor/file-existence-action@v2
-      with:
-        files: qmk_firmware
-
-    - name: Checkout QMK Firmware
-      uses: actions/checkout@v3
-      if: steps.check_files.outputs.files_exists == 'false'
-      with:
-        path: qmk_firmware
-        repository: ${{ inputs.qmk_repo || 'qmk/qmk_firmware' }}
-        ref: ${{ inputs.qmk_ref || 'master' }}
-        submodules: recursive
-        fetch-depth: 1000 # This line should be removed once userspace support exists in QMK Firmware
-
-    ########################################################
-    ## Delete this section once userspace exists upstream ##
-    ########################################################
-    - name: Patch QMK Firmware with userspace support
-      run: |
-        cd qmk_firmware
-        git config --global user.email "nonexistent@email-address.invalid"
-        git config --global user.name "QMK GitHub Actions User"
-        git remote add qmk https://github.com/qmk/qmk_firmware.git
-        git fetch qmk qmk-userspace
-        git merge --no-commit --squash qmk/qmk-userspace
-        cd ..
-
-    - name: Install QMK CLI
-      run: |
-        python3 -m pip install --upgrade qmk
-        python3 -m pip install -r qmk_firmware/requirements.txt
-
-    - name: Configure QMK CLI
-      run: |
-        qmk config mass_compile.parallel=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
-        qmk config user.qmk_home=$GITHUB_WORKSPACE/qmk_firmware
-        qmk config user.overlay_dir=$GITHUB_WORKSPACE
-
-    - name: Build
-      run: |
-        qmk mass-compile -e DUMP_CI_METADATA=yes -km ${{ github.repository_owner }} || touch .failed
-        # Generate the step summary markdown
-        ./qmk_firmware/util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
-        # Truncate to a maximum of 1MB to deal with GitHub workflow limit
-        truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
-        # Exit with failure if the compilation stage failed
-        [ ! -f .failed ] || exit 1
-
-    - name: Upload binaries
-      uses: actions/upload-artifact@v3
-      if: always() && !cancelled()
-      continue-on-error: true
-      with:
-        name: Firmware
-        path: |
-          *.bin
-          *.hex
-          *.uf2
-
-  publish:
-    name: Publish
-    needs: build
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Download binaries
-      if: always() && !cancelled()
-      uses: actions/download-artifact@v3
-
-    - name: Generate Release
-      uses: softprops/action-gh-release@v1
-      if: always() && !cancelled()
-      with:
-        token: "${{ github.token }}"
-        name: Latest Firmware
-        tag_name: latest
-        fail_on_unmatched_files: false
-        draft: false
-        prerelease: false
-        files: |
-          **/*.hex
-          **/*.bin
-          **/*.uf2

From dc86e605dd86e7e0051c5af2a03b0ae855a51e0b Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Thu, 7 Sep 2023 15:29:49 +1000
Subject: [PATCH 11/84] Add userspace for json builds

---
 lib/python/qmk/commands.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index ad73fc58d16e..4017f901eae7 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -197,6 +197,9 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=Fa
         'QMK_BIN="qmk"',
     ])
 
+    if HAS_QMK_USERSPACE:
+        make_command.append(f'QMK_USERSPACE={Path(QMK_USERSPACE).resolve()}')
+
     for key, value in env_vars.items():
         make_command.append(f'{key}={value}')
 

From 95d677920cfaff73204d07423bb6c431711e11a5 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 10 Sep 2023 12:36:26 +1000
Subject: [PATCH 12/84] Consistency in naming.

---
 Makefile                  | 2 +-
 builddefs/common_rules.mk | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index c6c2fe4956f4..d5404286b1a1 100644
--- a/Makefile
+++ b/Makefile
@@ -445,7 +445,7 @@ ifneq ($(QMK_USERSPACE),)
 .PHONY: distclean_userspace
 distclean: distclean_userspace
 distclean_userspace: clean
-	echo -n 'Deleting user overlay *.bin, *.hex, and *.uf2 ... '
+	echo -n 'Deleting userspace *.bin, *.hex, and *.uf2 ... '
 	rm -f $(QMK_USERSPACE)/*.bin $(QMK_USERSPACE)/*.hex $(QMK_USERSPACE)/*.uf2
 	echo 'done.'
 endif
diff --git a/builddefs/common_rules.mk b/builddefs/common_rules.mk
index ae17ad127e96..d6fd206e875d 100644
--- a/builddefs/common_rules.mk
+++ b/builddefs/common_rules.mk
@@ -202,7 +202,7 @@ cpfirmware: cpfirmware_qmk
 ifneq ($(QMK_USERSPACE),)
 cpfirmware: cpfirmware_userspace
 cpfirmware_userspace: cpfirmware_qmk
-	$(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to user overlay folder" | $(AWK_CMD)
+	$(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to userspace folder" | $(AWK_CMD)
 	$(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(QMK_USERSPACE)/$(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK)
 endif
 

From a649e8a27f20e77fc189fdc7fd8b89324414cf5b Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 15 Sep 2023 07:45:29 +1000
Subject: [PATCH 13/84] `qmk.json` and associated validation schema.

---
 data/schemas/user_repo.jsonschema | 26 ++++++++++++++++++++++
 lib/python/qmk/constants.py       | 36 +++++++++++++++++++++++++------
 2 files changed, 55 insertions(+), 7 deletions(-)
 create mode 100644 data/schemas/user_repo.jsonschema

diff --git a/data/schemas/user_repo.jsonschema b/data/schemas/user_repo.jsonschema
new file mode 100644
index 000000000000..c8956b3e6e93
--- /dev/null
+++ b/data/schemas/user_repo.jsonschema
@@ -0,0 +1,26 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema#",
+    "$id": "qmk.user_repo.v1",
+    "title": "User Repository Information",
+    "type": "object",
+    "required": [
+        "version",
+        "build_targets"
+    ],
+    "properties": {
+        "version": {
+            "type": "string"
+        },
+        "build_targets": {
+            "type": "array",
+            "items": {
+                "type": "array",
+                "items": {
+                    "type": "string",
+                    "minLength": 1,
+                    "pattern": "^[0-9a-z][0-9a-z_]*$"
+                }
+            }
+        }
+    }
+}
diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index 6d6c3dedda03..b3ef8a3eb887 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -6,28 +6,50 @@
 
 from milc import cli
 
+import jsonschema
+from qmk.json_schema import validate, json_load
+
 # The root of the qmk_firmware tree.
 QMK_FIRMWARE = Path.cwd()
 
 
 # Helper to detect userspace
 def _detect_qmk_userspace():
-    # If we're already in a directory with a Makefile and a keyboards or layouts directory, interpret it as userspace
+    def _validate_qmk_json(path):
+        try:
+            if (path / 'qmk.json').is_file():
+                qmkjson = json_load(path / 'qmk.json')
+                validate(qmkjson, 'qmk.user_repo.v1')
+                return True
+        except jsonschema.ValidationError:
+            pass
+        return False
+
+    test_dirs = []
+
+    # If we're already in a directory with a qmk.json and a keyboards or layouts directory, interpret it as userspace
     current_dir = Path(environ['ORIG_CWD'])
     while len(current_dir.parts) > 1:
-        if (current_dir / 'Makefile').is_file() and ((current_dir / 'keyboards').is_dir() or (current_dir / 'layouts').is_dir()):
-            if current_dir.resolve() != Path(QMK_FIRMWARE).resolve():
-                return current_dir
+        if (current_dir / 'qmk.json').is_file():
+            test_dirs.append(current_dir)
         current_dir = current_dir.parent
-    # Otherwise, use the environment variable or the configured default
-    return environ.get('QMK_USERSPACE') or cli.config.user.overlay_dir or QMK_FIRMWARE
+
+    test_dirs.append(environ.get('QMK_USERSPACE'))
+    test_dirs.append(cli.config.user.overlay_dir)
+    test_dirs = list([Path(x) for x in filter(lambda x: x is not None and Path(x).is_dir(), test_dirs)])
+
+    for test_dir in test_dirs:
+        if _validate_qmk_json(test_dir):
+            return test_dir
+
+    return None
 
 
 # The detected userspace tree
 QMK_USERSPACE = _detect_qmk_userspace()
 
 # Whether or not we have a separate userspace directory
-HAS_QMK_USERSPACE = True if Path(QMK_USERSPACE).resolve() != Path(QMK_FIRMWARE).resolve() else False
+HAS_QMK_USERSPACE = True if QMK_USERSPACE is not None else False
 
 # Upstream repo url
 QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'

From 4c4efa90570babe6865d1c6750a9340dc044274d Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 16 Sep 2023 23:56:30 +1000
Subject: [PATCH 14/84] Userspace CLI stubs.

---
 lib/python/qmk/cli/__init__.py           |  5 +++
 lib/python/qmk/cli/userspace/__init__.py |  5 +++
 lib/python/qmk/cli/userspace/add.py      |  5 +++
 lib/python/qmk/cli/userspace/build.py    |  5 +++
 lib/python/qmk/cli/userspace/doctor.py   | 17 ++++++++++
 lib/python/qmk/cli/userspace/list.py     |  5 +++
 lib/python/qmk/cli/userspace/remove.py   |  5 +++
 lib/python/qmk/constants.py              | 40 ++---------------------
 lib/python/qmk/userspace.py              | 41 ++++++++++++++++++++++++
 9 files changed, 90 insertions(+), 38 deletions(-)
 create mode 100644 lib/python/qmk/cli/userspace/__init__.py
 create mode 100644 lib/python/qmk/cli/userspace/add.py
 create mode 100644 lib/python/qmk/cli/userspace/build.py
 create mode 100644 lib/python/qmk/cli/userspace/doctor.py
 create mode 100644 lib/python/qmk/cli/userspace/list.py
 create mode 100644 lib/python/qmk/cli/userspace/remove.py
 create mode 100644 lib/python/qmk/userspace.py

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 9c3decf4f76a..9e4c2ec89032 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -79,6 +79,11 @@
     'qmk.cli.new.keymap',
     'qmk.cli.painter',
     'qmk.cli.pytest',
+    'qmk.cli.userspace.add',
+    'qmk.cli.userspace.build',
+    'qmk.cli.userspace.doctor',
+    'qmk.cli.userspace.list',
+    'qmk.cli.userspace.remove',
     'qmk.cli.via2json',
 ]
 
diff --git a/lib/python/qmk/cli/userspace/__init__.py b/lib/python/qmk/cli/userspace/__init__.py
new file mode 100644
index 000000000000..c2ac6576aa8c
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/__init__.py
@@ -0,0 +1,5 @@
+from . import doctor
+from . import add
+from . import remove
+from . import list
+from . import build
diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
new file mode 100644
index 000000000000..e220392c67a4
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -0,0 +1,5 @@
+from milc import cli
+
+@cli.subcommand('Adds a build target to userspace `qmk.json`.')
+def userspace_add(cli):
+    pass
\ No newline at end of file
diff --git a/lib/python/qmk/cli/userspace/build.py b/lib/python/qmk/cli/userspace/build.py
new file mode 100644
index 000000000000..4e8f535ef49d
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/build.py
@@ -0,0 +1,5 @@
+from milc import cli
+
+@cli.subcommand('Builds the build targets specified in userspace `qmk.json`.')
+def userspace_build(cli):
+    pass
\ No newline at end of file
diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py
new file mode 100644
index 000000000000..300c5e9575e2
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/doctor.py
@@ -0,0 +1,17 @@
+from milc import cli
+
+from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate
+
+@cli.subcommand('Checks userspace configuration.')
+def userspace_doctor(cli):
+    cli.log.info('Checking userspace configuration...')
+    cli.log.info(f'Detected QMK Firmware directory: {{fg_cyan}}{QMK_FIRMWARE}{{fg_reset}}')
+    cli.log.info(f'Detected QMK Userspace directory: {{fg_cyan}}{QMK_USERSPACE}{{fg_reset}}')
+    cli.log.info(f'QMK Userspace is enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}{{fg_reset}}')
+
+    for path in qmk_userspace_paths():
+        if qmk_userspace_validate(path):
+            cli.log.info(f'Testing {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`{{fg_reset}}')
+        else:
+            cli.log.warn(f'Testing {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`{{fg_reset}}')
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
new file mode 100644
index 000000000000..da3046ea1c2b
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -0,0 +1,5 @@
+from milc import cli
+
+@cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
+def userspace_list(cli):
+    pass
\ No newline at end of file
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
new file mode 100644
index 000000000000..7c9462104674
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -0,0 +1,5 @@
+from milc import cli
+
+@cli.subcommand('Removes a build target from userspace `qmk.json`.')
+def userspace_remove(cli):
+    pass
\ No newline at end of file
diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index b3ef8a3eb887..afa6d8af7ad0 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -4,49 +4,13 @@
 from datetime import date
 from pathlib import Path
 
-from milc import cli
-
-import jsonschema
-from qmk.json_schema import validate, json_load
+from qmk.userspace import detect_qmk_userspace
 
 # The root of the qmk_firmware tree.
 QMK_FIRMWARE = Path.cwd()
 
-
-# Helper to detect userspace
-def _detect_qmk_userspace():
-    def _validate_qmk_json(path):
-        try:
-            if (path / 'qmk.json').is_file():
-                qmkjson = json_load(path / 'qmk.json')
-                validate(qmkjson, 'qmk.user_repo.v1')
-                return True
-        except jsonschema.ValidationError:
-            pass
-        return False
-
-    test_dirs = []
-
-    # If we're already in a directory with a qmk.json and a keyboards or layouts directory, interpret it as userspace
-    current_dir = Path(environ['ORIG_CWD'])
-    while len(current_dir.parts) > 1:
-        if (current_dir / 'qmk.json').is_file():
-            test_dirs.append(current_dir)
-        current_dir = current_dir.parent
-
-    test_dirs.append(environ.get('QMK_USERSPACE'))
-    test_dirs.append(cli.config.user.overlay_dir)
-    test_dirs = list([Path(x) for x in filter(lambda x: x is not None and Path(x).is_dir(), test_dirs)])
-
-    for test_dir in test_dirs:
-        if _validate_qmk_json(test_dir):
-            return test_dir
-
-    return None
-
-
 # The detected userspace tree
-QMK_USERSPACE = _detect_qmk_userspace()
+QMK_USERSPACE = detect_qmk_userspace()
 
 # Whether or not we have a separate userspace directory
 HAS_QMK_USERSPACE = True if QMK_USERSPACE is not None else False
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
new file mode 100644
index 000000000000..e7a39ce05e1c
--- /dev/null
+++ b/lib/python/qmk/userspace.py
@@ -0,0 +1,41 @@
+from os import environ
+from pathlib import Path
+import jsonschema
+
+from milc import cli
+
+from qmk.json_schema import validate, json_load
+
+def qmk_userspace_paths():
+    test_dirs = []
+
+    # If we're already in a directory with a qmk.json and a keyboards or layouts directory, interpret it as userspace
+    current_dir = Path(environ['ORIG_CWD'])
+    while len(current_dir.parts) > 1:
+        if (current_dir / 'qmk.json').is_file():
+            test_dirs.append(current_dir)
+        current_dir = current_dir.parent
+
+    test_dirs.append(environ.get('QMK_USERSPACE'))
+    test_dirs.append(cli.config.user.overlay_dir)
+    test_dirs = list(dict.fromkeys([Path(x) for x in filter(lambda x: x is not None and Path(x).is_dir(), test_dirs)]))
+    return test_dirs
+
+
+def qmk_userspace_validate(path):
+    try:
+        if (path / 'qmk.json').is_file():
+            qmkjson = json_load(path / 'qmk.json')
+            validate(qmkjson, 'qmk.user_repo.v1')
+            return True
+    except jsonschema.ValidationError:
+        pass
+    return False
+
+
+def detect_qmk_userspace():
+    test_dirs = qmk_userspace_paths()
+    for test_dir in test_dirs:
+        if qmk_userspace_validate(test_dir):
+            return test_dir
+    return None

From 4f923b5ee7c03dded78281891e41176d0f3e9154 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 17 Sep 2023 00:01:50 +1000
Subject: [PATCH 15/84] Formatting.

---
 lib/python/qmk/cli/userspace/add.py    |  3 ++-
 lib/python/qmk/cli/userspace/build.py  |  3 ++-
 lib/python/qmk/cli/userspace/doctor.py | 14 +++-----------
 lib/python/qmk/cli/userspace/list.py   |  3 ++-
 lib/python/qmk/cli/userspace/remove.py |  3 ++-
 lib/python/qmk/userspace.py            | 14 ++++++++++++++
 6 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index e220392c67a4..e7a4aab17fba 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -1,5 +1,6 @@
 from milc import cli
 
+
 @cli.subcommand('Adds a build target to userspace `qmk.json`.')
 def userspace_add(cli):
-    pass
\ No newline at end of file
+    pass
diff --git a/lib/python/qmk/cli/userspace/build.py b/lib/python/qmk/cli/userspace/build.py
index 4e8f535ef49d..e6101b297f64 100644
--- a/lib/python/qmk/cli/userspace/build.py
+++ b/lib/python/qmk/cli/userspace/build.py
@@ -1,5 +1,6 @@
 from milc import cli
 
+
 @cli.subcommand('Builds the build targets specified in userspace `qmk.json`.')
 def userspace_build(cli):
-    pass
\ No newline at end of file
+    pass
diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py
index 300c5e9575e2..8c1b95f50231 100644
--- a/lib/python/qmk/cli/userspace/doctor.py
+++ b/lib/python/qmk/cli/userspace/doctor.py
@@ -1,17 +1,9 @@
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
-from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate
+from qmk.userspace import userspace_doctor_checks
+
 
 @cli.subcommand('Checks userspace configuration.')
 def userspace_doctor(cli):
-    cli.log.info('Checking userspace configuration...')
-    cli.log.info(f'Detected QMK Firmware directory: {{fg_cyan}}{QMK_FIRMWARE}{{fg_reset}}')
-    cli.log.info(f'Detected QMK Userspace directory: {{fg_cyan}}{QMK_USERSPACE}{{fg_reset}}')
-    cli.log.info(f'QMK Userspace is enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}{{fg_reset}}')
-
-    for path in qmk_userspace_paths():
-        if qmk_userspace_validate(path):
-            cli.log.info(f'Testing {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`{{fg_reset}}')
-        else:
-            cli.log.warn(f'Testing {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`{{fg_reset}}')
+    userspace_doctor_checks(QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE)
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index da3046ea1c2b..35d23c90004f 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -1,5 +1,6 @@
 from milc import cli
 
+
 @cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
 def userspace_list(cli):
-    pass
\ No newline at end of file
+    pass
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
index 7c9462104674..efa24c2772bf 100644
--- a/lib/python/qmk/cli/userspace/remove.py
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -1,5 +1,6 @@
 from milc import cli
 
+
 @cli.subcommand('Removes a build target from userspace `qmk.json`.')
 def userspace_remove(cli):
-    pass
\ No newline at end of file
+    pass
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index e7a39ce05e1c..09bbb142b333 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -6,6 +6,7 @@
 
 from qmk.json_schema import validate, json_load
 
+
 def qmk_userspace_paths():
     test_dirs = []
 
@@ -39,3 +40,16 @@ def detect_qmk_userspace():
         if qmk_userspace_validate(test_dir):
             return test_dir
     return None
+
+
+def userspace_doctor_checks(qmk_firmware, qmk_userspace, has_qmk_userspace):
+    cli.log.info('Checking userspace configuration...')
+    cli.log.info(f'Detected QMK Firmware directory: {{fg_cyan}}{qmk_firmware}{{fg_reset}}')
+    cli.log.info(f'Detected QMK Userspace directory: {{fg_cyan}}{qmk_userspace}{{fg_reset}}')
+    cli.log.info(f'QMK Userspace is enabled: {{fg_cyan}}{has_qmk_userspace}{{fg_reset}}')
+
+    for path in qmk_userspace_paths():
+        if qmk_userspace_validate(path):
+            cli.log.info(f'Testing {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`{{fg_reset}}')
+        else:
+            cli.log.warn(f'Testing {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`{{fg_reset}}')

From 7502772488373c76544fdc52f99872f6502290d2 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 17 Sep 2023 00:06:04 +1000
Subject: [PATCH 16/84] Hook up `qmk userspace-doctor` with `qmk doctor`

---
 lib/python/qmk/cli/doctor/main.py |  6 +++++-
 lib/python/qmk/userspace.py       | 11 ++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 6a6feb87d10b..204607620f9b 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -9,10 +9,11 @@
 from milc.questions import yesno
 
 from qmk import submodules
-from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM
+from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM, QMK_USERSPACE, HAS_QMK_USERSPACE
 from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
 from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation
 from qmk.commands import in_virtualenv
+from qmk.userspace import userspace_doctor_checks
 
 
 def os_tests():
@@ -108,6 +109,9 @@ def doctor(cli):
     cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE)
 
     status = os_status = os_tests()
+
+    userspace_doctor_checks(None, QMK_USERSPACE, HAS_QMK_USERSPACE)
+
     git_status = git_tests()
 
     if git_status == CheckStatus.ERROR or (os_status == CheckStatus.OK and git_status == CheckStatus.WARNING):
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index 09bbb142b333..7dddb702ecf0 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -43,13 +43,14 @@ def detect_qmk_userspace():
 
 
 def userspace_doctor_checks(qmk_firmware, qmk_userspace, has_qmk_userspace):
-    cli.log.info('Checking userspace configuration...')
-    cli.log.info(f'Detected QMK Firmware directory: {{fg_cyan}}{qmk_firmware}{{fg_reset}}')
+    if qmk_firmware:
+        cli.log.info(f'Detected QMK Firmware directory: {{fg_cyan}}{qmk_firmware}{{fg_reset}}')
     cli.log.info(f'Detected QMK Userspace directory: {{fg_cyan}}{qmk_userspace}{{fg_reset}}')
-    cli.log.info(f'QMK Userspace is enabled: {{fg_cyan}}{has_qmk_userspace}{{fg_reset}}')
 
     for path in qmk_userspace_paths():
         if qmk_userspace_validate(path):
-            cli.log.info(f'Testing {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`{{fg_reset}}')
+            cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`{{fg_reset}}')
         else:
-            cli.log.warn(f'Testing {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`{{fg_reset}}')
+            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`{{fg_reset}}')
+
+    cli.log.info(f'QMK Userspace is enabled: {{fg_cyan}}{has_qmk_userspace}{{fg_reset}}')

From e12d3e274e8171e174397d9d8b51a5a7bcf12a3d Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 17 Sep 2023 00:17:33 +1000
Subject: [PATCH 17/84] `qmk pytest`

---
 lib/python/qmk/keymap.py |  2 +-
 lib/python/qmk/path.py   | 10 ++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index f3c620ed8ef8..340425740f2d 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -491,7 +491,7 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
 
     # walk up the directory tree until keyboards_dir
     # and collect all directories' name with keymap.c file in it
-    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE]:
+    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]:
         keyboards_dir = search_dir / Path('keyboards')
         kb_path = keyboards_dir / keyboard
 
diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py
index 3a069e2c15dd..7a6655ff4727 100644
--- a/lib/python/qmk/path.py
+++ b/lib/python/qmk/path.py
@@ -5,7 +5,7 @@
 import argparse
 from pathlib import Path
 
-from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE, QMK_USERSPACE
+from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.errors import NoSuchKeyboardError
 
 
@@ -36,9 +36,11 @@ def under_qmk_userspace():
     cwd = Path(os.environ['ORIG_CWD'])
 
     try:
-        return cwd.relative_to(QMK_USERSPACE)
+        if HAS_QMK_USERSPACE:
+            return cwd.relative_to(QMK_USERSPACE)
     except ValueError:
-        return None
+        pass
+    return None
 
 
 def keyboard(keyboard_name):
@@ -58,7 +60,7 @@ def keymaps(keyboard_name):
     keyboard_folder = keyboard(keyboard_name)
     found_dirs = []
 
-    for root_dir in [QMK_USERSPACE, QMK_FIRMWARE]:
+    for root_dir in [QMK_USERSPACE, QMK_FIRMWARE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]:
         this_keyboard_folder = root_dir / keyboard_folder
         for _ in range(MAX_KEYBOARD_SUBFOLDERS):
             if (this_keyboard_folder / 'keymaps').exists():

From 946ae03027a654999f712f1b4273f588111bfaf3 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 17 Sep 2023 00:22:22 +1000
Subject: [PATCH 18/84] Sequencing.

---
 lib/python/qmk/userspace.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index 7dddb702ecf0..e53408818385 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -45,7 +45,6 @@ def detect_qmk_userspace():
 def userspace_doctor_checks(qmk_firmware, qmk_userspace, has_qmk_userspace):
     if qmk_firmware:
         cli.log.info(f'Detected QMK Firmware directory: {{fg_cyan}}{qmk_firmware}{{fg_reset}}')
-    cli.log.info(f'Detected QMK Userspace directory: {{fg_cyan}}{qmk_userspace}{{fg_reset}}')
 
     for path in qmk_userspace_paths():
         if qmk_userspace_validate(path):
@@ -53,4 +52,5 @@ def userspace_doctor_checks(qmk_firmware, qmk_userspace, has_qmk_userspace):
         else:
             cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`{{fg_reset}}')
 
+    cli.log.info(f'Selected QMK Userspace directory: {{fg_cyan}}{qmk_userspace}{{fg_reset}}')
     cli.log.info(f'QMK Userspace is enabled: {{fg_cyan}}{has_qmk_userspace}{{fg_reset}}')

From b9df68dfeb338ccefb6a3af4d9e3119e05c1066e Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 22 Sep 2023 16:26:52 +1000
Subject: [PATCH 19/84] Direct compilation.

---
 builddefs/build_keyboard.mk            |  7 ++-
 data/schemas/user_repo.jsonschema      |  6 +-
 lib/python/qmk/cli/compile.py          |  9 +--
 lib/python/qmk/cli/doctor/main.py      | 19 +++++-
 lib/python/qmk/cli/format/json.py      | 25 +++++---
 lib/python/qmk/cli/mass_compile.py     | 14 +++--
 lib/python/qmk/cli/userspace/doctor.py |  6 +-
 lib/python/qmk/commands.py             | 85 ++++++++++++++++++--------
 lib/python/qmk/json_encoders.py        | 18 ++++++
 lib/python/qmk/userspace.py            | 14 -----
 10 files changed, 139 insertions(+), 64 deletions(-)

diff --git a/builddefs/build_keyboard.mk b/builddefs/build_keyboard.mk
index 998cdd66cda9..7e8e8a83c753 100644
--- a/builddefs/build_keyboard.mk
+++ b/builddefs/build_keyboard.mk
@@ -384,7 +384,8 @@ ifeq ("$(USER_NAME)","")
 endif
 USER_PATH := users/$(USER_NAME)
 
-ifneq ($(QMK_USERSPACE),)
+# If the equivalent users directory exists in userspace, use that in preference to anything currently in the main repo
+ifneq ($(wildcard $(QMK_USERSPACE)/$(USER_PATH)),)
     USER_PATH := $(QMK_USERSPACE)/$(USER_PATH)
 endif
 
@@ -428,6 +429,10 @@ ifneq ("$(KEYMAP_H)","")
     CONFIG_H += $(KEYMAP_H)
 endif
 
+ifeq ($(KEYMAP_C),)
+    $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap)
+endif
+
 OPT_DEFS += -DKEYMAP_C=\"$(KEYMAP_C)\"
 
 # If a keymap or userspace places their keymap array in another file instead, allow for it to be included
diff --git a/data/schemas/user_repo.jsonschema b/data/schemas/user_repo.jsonschema
index c8956b3e6e93..64d772962da6 100644
--- a/data/schemas/user_repo.jsonschema
+++ b/data/schemas/user_repo.jsonschema
@@ -4,17 +4,19 @@
     "title": "User Repository Information",
     "type": "object",
     "required": [
-        "version",
+        "userspace_version",
         "build_targets"
     ],
     "properties": {
-        "version": {
+        "userspace_version": {
             "type": "string"
         },
         "build_targets": {
             "type": "array",
             "items": {
                 "type": "array",
+                "minItems": 2,
+                "maxItems": 2,
                 "items": {
                     "type": "string",
                     "minLength": 1,
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index f43e5f32dea9..ee36e8489d96 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -8,7 +8,7 @@
 
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
+from qmk.commands import compile_configurator_json, create_make_command, create_make_target, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
 from qmk.keymap import keymap_completer, locate_keymap
 
@@ -30,6 +30,7 @@ def _is_keymap_target(keyboard, keymap):
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
+@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")
 @cli.subcommand('Compile a QMK Firmware.')
 @automagic_keyboard
 @automagic_keymap
@@ -49,7 +50,7 @@ def compile(cli):
     if cli.args.filename:
         # If a configurator JSON was provided generate a keymap and compile it
         user_keymap = parse_configurator_json(cli.args.filename)
-        commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
+        commands = [compile_configurator_json(user_keymap, target=cli.args.target, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
 
     elif cli.config.compile.keyboard and cli.config.compile.keymap:
         # Generate the make command for a specific keyboard/keymap.
@@ -59,8 +60,8 @@ def compile(cli):
             return False
 
         if cli.args.clean:
-            commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
-        commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
+            commands.append(create_make_target('clean'))
+        commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, target=cli.args.target, parallel=cli.config.compile.parallel, **envs))
 
     if not commands:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 204607620f9b..166ceb95b2b8 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -13,7 +13,7 @@
 from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
 from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation
 from qmk.commands import in_virtualenv
-from qmk.userspace import userspace_doctor_checks
+from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate
 
 
 def os_tests():
@@ -93,6 +93,21 @@ def output_submodule_status():
                 cli.log.error(f'- {sub_name}: <<< missing or unknown >>>')
 
 
+def userspace_tests(qmk_firmware):
+    if qmk_firmware:
+        cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}')
+
+    for path in qmk_userspace_paths():
+        if qmk_userspace_validate(path):
+            cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`')
+        else:
+            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`')
+
+    if QMK_USERSPACE is not None:
+        cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}')
+    cli.log.info(f'Userspace enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}')
+
+
 @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.')
 @cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.')
 @cli.subcommand('Basic QMK environment checks')
@@ -110,7 +125,7 @@ def doctor(cli):
 
     status = os_status = os_tests()
 
-    userspace_doctor_checks(None, QMK_USERSPACE, HAS_QMK_USERSPACE)
+    userspace_tests(None)
 
     git_status = git_tests()
 
diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py
index 3299a0d80704..4f6b6d9e79d6 100755
--- a/lib/python/qmk/cli/format/json.py
+++ b/lib/python/qmk/cli/format/json.py
@@ -9,12 +9,12 @@
 
 from qmk.info import info_json
 from qmk.json_schema import json_load, validate
-from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder
+from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder
 from qmk.path import normpath
 
 
 @cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
-@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
+@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
 @cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
 @cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
 @cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
@@ -24,18 +24,27 @@ def format_json(cli):
     json_file = json_load(cli.args.json_file)
 
     if cli.args.format == 'auto':
+        json_encoder = None
         try:
-            validate(json_file, 'qmk.keyboard.v1')
-            json_encoder = InfoJSONEncoder
-
+            validate(json_file, 'qmk.user_repo.v1')
+            json_encoder = UserspaceJSONEncoder
         except ValidationError as e:
-            cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e)
-            cli.log.info('Treating %s as a keymap file.', cli.args.json_file)
-            json_encoder = KeymapJSONEncoder
+            pass
+
+        if json_encoder is None:
+            try:
+                validate(json_file, 'qmk.keyboard.v1')
+                json_encoder = InfoJSONEncoder
+            except ValidationError as e:
+                cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', cli.args.json_file, e)
+                cli.log.info('Treating %s as a keymap file.', cli.args.json_file)
+                json_encoder = KeymapJSONEncoder
     elif cli.args.format == 'keyboard':
         json_encoder = InfoJSONEncoder
     elif cli.args.format == 'keymap':
         json_encoder = KeymapJSONEncoder
+    elif cli.args.format == 'userspace':
+        json_encoder = UserspaceJSONEncoder
     else:
         # This should be impossible
         cli.log.error('Unknown format: %s', cli.args.format)
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index b978a1ef7ced..2168bf0c524c 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -8,7 +8,8 @@
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
-from qmk.commands import _find_make, get_make_parallel_args
+from qmk.commands import _find_make, get_make_parallel_args, compile_configurator_json, create_make_target, create_make_command, parse_configurator_json, build_environment
+
 from qmk.keyboard import resolve_keyboard
 from qmk.search import search_keymap_targets
 
@@ -34,7 +35,7 @@ def mass_compile(cli):
     """
     make_cmd = _find_make()
     if cli.args.clean:
-        cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
+        cli.run(create_make_target('clean'), capture_output=False, stdin=DEVNULL)
 
     builddir = Path(QMK_FIRMWARE) / '.build'
     makefile = builddir / 'parallel_kb_builds.mk'
@@ -50,9 +51,7 @@ def mass_compile(cli):
     builddir.mkdir(parents=True, exist_ok=True)
     with open(makefile, "w") as f:
 
-        userspace_suffix = ''
-        if HAS_QMK_USERSPACE:
-            userspace_suffix = f'QMK_USERSPACE={Path(QMK_USERSPACE).resolve()}'
+        envs = build_environment(cli.args.env)
 
         for target in sorted(targets):
             keyboard_name = target[0]
@@ -60,6 +59,9 @@ def mass_compile(cli):
             keyboard_safe = keyboard_name.replace('/', '_')
             build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
             failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
+
+            make_command = create_make_command(keyboard_name, keymap_name, make_override='+@$(MAKE)', **envs)
+            command_text = ' '.join(make_command)
             # yapf: disable
             f.write(
                 f"""\
@@ -67,7 +69,7 @@ def mass_compile(cli):
 {keyboard_safe}_{keymap_name}_binary:
 	@rm -f "{build_log}" || true
 	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {userspace_suffix} {' '.join(cli.args.env)} \\
+	{command_text} \\
 		>>"{build_log}" 2>&1 \\
 		|| cp "{build_log}" "{failed_log}"
 	@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py
index 8c1b95f50231..8f6a08161e6e 100644
--- a/lib/python/qmk/cli/userspace/doctor.py
+++ b/lib/python/qmk/cli/userspace/doctor.py
@@ -1,9 +1,9 @@
 from milc import cli
 
-from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
-from qmk.userspace import userspace_doctor_checks
+from qmk.constants import QMK_FIRMWARE
+from qmk.cli.doctor.main import userspace_tests
 
 
 @cli.subcommand('Checks userspace configuration.')
 def userspace_doctor(cli):
-    userspace_doctor_checks(QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE)
+    userspace_tests(QMK_FIRMWARE)
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 4017f901eae7..5ce9dcf31b98 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -9,13 +9,16 @@
 from milc import cli
 import jsonschema
 
-from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX, QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX, QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.json_schema import json_load, validate
 
 
-def _find_make():
+def _find_make(make_override=None):
     """Returns the correct make command for this environment.
     """
+    if make_override:
+        return make_override
+
     make_cmd = os.environ.get('MAKE')
 
     if not make_cmd:
@@ -24,7 +27,7 @@ def _find_make():
     return make_cmd
 
 
-def create_make_target(target, dry_run=False, parallel=1, **env_vars):
+def create_make_target(target, dry_run=False, parallel=1, make_override=None, **env_vars):
     """Create a make command
 
     Args:
@@ -46,7 +49,7 @@ def create_make_target(target, dry_run=False, parallel=1, **env_vars):
         A command that can be run to make the specified keyboard and keymap
     """
     env = []
-    make_cmd = _find_make()
+    make_cmd = _find_make(make_override=make_override)
 
     for key, value in env_vars.items():
         env.append(f'{key}={value}')
@@ -60,7 +63,7 @@ def create_make_target(target, dry_run=False, parallel=1, **env_vars):
     return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]
 
 
-def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars):
+def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, make_override=None, **env_vars):
     """Create a make compile command
 
     Args:
@@ -72,7 +75,7 @@ def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1
             The name of the keymap, for example 'algernon'
 
         target
-            Usually a bootloader.
+            The build target to compile; usually the bootloader parameter to flash.
 
         dry_run
             make -n -- don't actually build
@@ -87,12 +90,43 @@ def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1
 
         A command that can be run to make the specified keyboard and keymap
     """
-    make_args = [keyboard, keymap]
+    # Return a command that can be run to make the keymap and flash if given
+    make_command = [_find_make(make_override=make_override)]
+
+    if not cli.config.general.verbose:
+        make_command.append('-s')
+
+    if dry_run:
+        make_command.append('-n')
+
+    make_command.extend([
+        *get_make_parallel_args(parallel),
+        '-r',
+        '-R',
+        '-f',
+        'builddefs/build_keyboard.mk',
+    ])
 
     if target:
-        make_args.append(target)
+        make_command.append(target)
+
+    make_command.extend([
+        f'KEYBOARD={keyboard}',
+        f'KEYMAP={keymap}',
+        'SILENT=false',
+        'QMK_BIN="qmk"',
+    ])
+
+    if cli.config.general.verbose:
+        make_command.append('VERBOSE=true')
+
+    if HAS_QMK_USERSPACE:
+        make_command.append(f'QMK_USERSPACE={Path(QMK_USERSPACE).resolve()}')
 
-    return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars)
+    for key, value in env_vars.items():
+        make_command.append(f'{key}={value}')
+
+    return make_command
 
 
 def get_make_parallel_args(parallel=1):
@@ -100,20 +134,19 @@ def get_make_parallel_args(parallel=1):
     """
     parallel_args = []
 
-    if int(parallel) <= 0:
-        # 0 or -1 means -j without argument (unlimited jobs)
-        parallel_args.append('--jobs')
-    else:
-        parallel_args.append('--jobs=' + str(parallel))
-
     if int(parallel) != 1:
-        # If more than 1 job is used, synchronize parallel output by target
+        if int(parallel) <= 0:
+            # 0 or -1 means -j without argument (unlimited jobs)
+            parallel_args.append('--jobs')
+        elif int(parallel) > 1:
+            parallel_args.append('--jobs=' + str(parallel))
+            # If more than 1 job is used, synchronize parallel output by target
         parallel_args.append('--output-sync=target')
 
     return parallel_args
 
 
-def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars):
+def compile_configurator_json(user_keymap, target=None, parallel=1, make_override=None, clean=False, **env_vars):
     """Convert a configurator export JSON file into a C file and then compile it.
 
     Args:
@@ -121,8 +154,8 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=Fa
         user_keymap
             A deserialized keymap export
 
-        bootloader
-            A bootloader to flash
+        target
+            The build target to compile; usually the bootloader parameter to flash.
 
         parallel
             The number of make jobs to run in parallel
@@ -131,12 +164,16 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=Fa
 
         A command to run to compile and flash the C file.
     """
+    # NOTE: this doesn't just generate the GNU Make command to execute, it also dumps the
+    #       keymap.json file to the correct location for the build system to find it.
+    #       Beware of using this func before executing a `make clean` -- it'll get nuked.
+    #
     # In case the user passes a keymap.json from a keymap directory directly to the CLI.
     # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
     user_keymap["keymap"] = user_keymap.get("keymap", "default_json")
 
     keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
-    target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
+    compile_target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
     intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{keyboard_filesafe}_{user_keymap["keymap"]}')
     keymap_dir = intermediate_output / 'src'
     keymap_json = keymap_dir / 'keymap.json'
@@ -162,7 +199,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=Fa
     # Return a command that can be run to make the keymap and flash if given
     verbose = 'true' if cli.config.general.verbose else 'false'
     color = 'true' if cli.config.general.color else 'false'
-    make_command = [_find_make()]
+    make_command = [_find_make(make_override)]
 
     if not cli.config.general.verbose:
         make_command.append('-s')
@@ -175,14 +212,14 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=Fa
         'builddefs/build_keyboard.mk',
     ])
 
-    if bootloader:
-        make_command.append(bootloader)
+    if target:
+        make_command.append(target)
 
     make_command.extend([
         f'KEYBOARD={user_keymap["keyboard"]}',
         f'KEYMAP={user_keymap["keymap"]}',
         f'KEYBOARD_FILESAFE={keyboard_filesafe}',
-        f'TARGET={target}',
+        f'TARGET={compile_target}',
         f'INTERMEDIATE_OUTPUT={intermediate_output}',
         f'MAIN_KEYMAP_PATH_1={intermediate_output}',
         f'MAIN_KEYMAP_PATH_2={intermediate_output}',
diff --git a/lib/python/qmk/json_encoders.py b/lib/python/qmk/json_encoders.py
index 1e90f6a28808..0e4ad1d22028 100755
--- a/lib/python/qmk/json_encoders.py
+++ b/lib/python/qmk/json_encoders.py
@@ -217,3 +217,21 @@ def sort_dict(self, item):
                 return '50' + str(key)
 
         return key
+
+
+class UserspaceJSONEncoder(QMKJSONEncoder):
+    """Custom encoder to make userspace qmk.json's a little nicer to work with.
+    """
+    def sort_dict(self, item):
+        """Sorts the hashes in a nice way.
+        """
+        key = item[0]
+
+        if self.indentation_level == 1:
+            if key == 'userspace_version':
+                return '00userspace_version'
+
+            if key == 'build_targets':
+                return '01build_targets'
+
+        return key
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index e53408818385..20a282b850fc 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -40,17 +40,3 @@ def detect_qmk_userspace():
         if qmk_userspace_validate(test_dir):
             return test_dir
     return None
-
-
-def userspace_doctor_checks(qmk_firmware, qmk_userspace, has_qmk_userspace):
-    if qmk_firmware:
-        cli.log.info(f'Detected QMK Firmware directory: {{fg_cyan}}{qmk_firmware}{{fg_reset}}')
-
-    for path in qmk_userspace_paths():
-        if qmk_userspace_validate(path):
-            cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`{{fg_reset}}')
-        else:
-            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`{{fg_reset}}')
-
-    cli.log.info(f'Selected QMK Userspace directory: {{fg_cyan}}{qmk_userspace}{{fg_reset}}')
-    cli.log.info(f'QMK Userspace is enabled: {{fg_cyan}}{has_qmk_userspace}{{fg_reset}}')

From 7bd2706b806025f6da0ce73b7a141d5455a4ca84 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 22 Sep 2023 17:14:40 +1000
Subject: [PATCH 20/84] Validation errors.

---
 data/schemas/user_repo_v0.jsonschema          | 14 +++++++++
 ...epo.jsonschema => user_repo_v1.jsonschema} |  3 +-
 lib/python/qmk/cli/doctor/main.py             | 10 ++++++-
 lib/python/qmk/userspace.py                   | 29 +++++++++++++++----
 4 files changed, 49 insertions(+), 7 deletions(-)
 create mode 100644 data/schemas/user_repo_v0.jsonschema
 rename data/schemas/{user_repo.jsonschema => user_repo_v1.jsonschema} (92%)

diff --git a/data/schemas/user_repo_v0.jsonschema b/data/schemas/user_repo_v0.jsonschema
new file mode 100644
index 000000000000..b18ac504284b
--- /dev/null
+++ b/data/schemas/user_repo_v0.jsonschema
@@ -0,0 +1,14 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema#",
+    "$id": "qmk.user_repo.v0",
+    "title": "User Repository Information",
+    "type": "object",
+    "required": [
+        "userspace_version"
+    ],
+    "properties": {
+        "userspace_version": {
+            "type": "string",
+        },
+    }
+}
diff --git a/data/schemas/user_repo.jsonschema b/data/schemas/user_repo_v1.jsonschema
similarity index 92%
rename from data/schemas/user_repo.jsonschema
rename to data/schemas/user_repo_v1.jsonschema
index 64d772962da6..751347365675 100644
--- a/data/schemas/user_repo.jsonschema
+++ b/data/schemas/user_repo_v1.jsonschema
@@ -9,7 +9,8 @@
     ],
     "properties": {
         "userspace_version": {
-            "type": "string"
+            "type": "string",
+            "enum": ["1.0"]
         },
         "build_targets": {
             "type": "array",
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 166ceb95b2b8..3576913321bf 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -98,10 +98,18 @@ def userspace_tests(qmk_firmware):
         cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}')
 
     for path in qmk_userspace_paths():
-        if qmk_userspace_validate(path):
+        validation_err = None
+
+        def validation_err_handler(err):
+            nonlocal validation_err
+            validation_err = err
+
+        if qmk_userspace_validate(path, validation_err_handler):
             cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`')
         else:
             cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`')
+            if validation_err is not None:
+                cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {validation_err}')
 
     if QMK_USERSPACE is not None:
         cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}')
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index 20a282b850fc..ad4b2f841ed7 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -23,13 +23,15 @@ def qmk_userspace_paths():
     return test_dirs
 
 
-def qmk_userspace_validate(path):
+def qmk_userspace_validate(path, validation_error_callback=None):
     try:
         if (path / 'qmk.json').is_file():
-            qmkjson = json_load(path / 'qmk.json')
-            validate(qmkjson, 'qmk.user_repo.v1')
-            return True
-    except jsonschema.ValidationError:
+            userspace_defs = UserspaceDefs(path / 'qmk.json')
+            if userspace_defs is not None:
+                return True
+    except jsonschema.ValidationError as err:
+        if validation_error_callback is not None:
+            validation_error_callback(err)
         pass
     return False
 
@@ -40,3 +42,20 @@ def detect_qmk_userspace():
         if qmk_userspace_validate(test_dir):
             return test_dir
     return None
+
+
+class UserspaceDefs:
+    def __init__(self, userspace_json: Path):
+        json = json_load(userspace_json)
+        validate(json, 'qmk.user_repo.v0')  # `qmk.json` must have a userspace_version at minimum
+
+        # TODO: validate different versions of json when we're ready to deal with versioning
+        # Start at highest version, then run down the list to v1
+        try:
+            validate(json, 'qmk.user_repo.v1')
+            self.__load_v1(json)
+        except jsonschema.ValidationError:
+            raise  # v1 always needs to raise here -- higher versions shouldn't bother and should just `pass`
+
+    def __load_v1(self, json):
+        self.json = json

From 90d211b9244d6d25328bf5cf455aa95d1e8a57f5 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 22 Sep 2023 17:31:26 +1000
Subject: [PATCH 21/84] Userspace `qmk.json` save API.

---
 lib/python/qmk/cli/userspace/add.py    |  9 ++++++++-
 lib/python/qmk/cli/userspace/build.py  |  9 ++++++++-
 lib/python/qmk/cli/userspace/list.py   |  9 ++++++++-
 lib/python/qmk/cli/userspace/remove.py |  9 ++++++++-
 lib/python/qmk/userspace.py            | 20 +++++++++++++++++++-
 5 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index e7a4aab17fba..e0f1de4bfbd1 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -1,6 +1,13 @@
 from milc import cli
 
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.userspace import UserspaceDefs
+
 
 @cli.subcommand('Adds a build target to userspace `qmk.json`.')
 def userspace_add(cli):
-    pass
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
\ No newline at end of file
diff --git a/lib/python/qmk/cli/userspace/build.py b/lib/python/qmk/cli/userspace/build.py
index e6101b297f64..373a13256ba2 100644
--- a/lib/python/qmk/cli/userspace/build.py
+++ b/lib/python/qmk/cli/userspace/build.py
@@ -1,6 +1,13 @@
 from milc import cli
 
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.userspace import UserspaceDefs
+
 
 @cli.subcommand('Builds the build targets specified in userspace `qmk.json`.')
 def userspace_build(cli):
-    pass
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
\ No newline at end of file
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index 35d23c90004f..9c0d0b705f6e 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -1,6 +1,13 @@
 from milc import cli
 
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.userspace import UserspaceDefs
+
 
 @cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
 def userspace_list(cli):
-    pass
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
\ No newline at end of file
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
index efa24c2772bf..aa244d800eb2 100644
--- a/lib/python/qmk/cli/userspace/remove.py
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -1,6 +1,13 @@
 from milc import cli
 
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.userspace import UserspaceDefs
+
 
 @cli.subcommand('Removes a build target from userspace `qmk.json`.')
 def userspace_remove(cli):
-    pass
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
\ No newline at end of file
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index ad4b2f841ed7..c3d43f0acbaf 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -1,10 +1,12 @@
 from os import environ
 from pathlib import Path
+import json
 import jsonschema
 
 from milc import cli
 
 from qmk.json_schema import validate, json_load
+from qmk.json_encoders import UserspaceJSONEncoder
 
 
 def qmk_userspace_paths():
@@ -46,6 +48,7 @@ def detect_qmk_userspace():
 
 class UserspaceDefs:
     def __init__(self, userspace_json: Path):
+        self.path = userspace_json
         json = json_load(userspace_json)
         validate(json, 'qmk.user_repo.v0')  # `qmk.json` must have a userspace_version at minimum
 
@@ -57,5 +60,20 @@ def __init__(self, userspace_json: Path):
         except jsonschema.ValidationError:
             raise  # v1 always needs to raise here -- higher versions shouldn't bother and should just `pass`
 
+    def save(self):
+        target_json = {
+            "userspace_version": "1.0",  # Needs to match latest version
+            "build_targets":
+                self.build_targets  # Only other field needed in v1
+        }
+
+        try:
+            # Ensure what we're writing validates against the latest version of the schema
+            validate(target_json, 'qmk.user_repo.v1')
+        except jsonschema.ValidationError as err:
+            cli.log.error(f'Could not save userspace file: {err}')
+
+        self.path.write_text(json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True))
+
     def __load_v1(self, json):
-        self.json = json
+        self.build_targets = json['build_targets']

From 6a5ffb11064c66a2e19792db31b8bdde26bd9ef7 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 22 Sep 2023 20:35:27 +1000
Subject: [PATCH 22/84] Collect validation errors when loading userspace.

---
 lib/python/qmk/cli/doctor/main.py | 16 +++------
 lib/python/qmk/userspace.py       | 58 ++++++++++++++++++++++---------
 2 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 3576913321bf..0c1e4bc0b3b5 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -13,7 +13,7 @@
 from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
 from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation
 from qmk.commands import in_virtualenv
-from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate
+from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationException
 
 
 def os_tests():
@@ -98,18 +98,12 @@ def userspace_tests(qmk_firmware):
         cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}')
 
     for path in qmk_userspace_paths():
-        validation_err = None
-
-        def validation_err_handler(err):
-            nonlocal validation_err
-            validation_err = err
-
-        if qmk_userspace_validate(path, validation_err_handler):
+        try:
+            qmk_userspace_validate(path)
             cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`')
-        else:
+        except UserspaceValidationException as err:
             cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`')
-            if validation_err is not None:
-                cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {validation_err}')
+            cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {err}')
 
     if QMK_USERSPACE is not None:
         cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}')
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index c3d43f0acbaf..8d050b2ca078 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -25,24 +25,19 @@ def qmk_userspace_paths():
     return test_dirs
 
 
-def qmk_userspace_validate(path, validation_error_callback=None):
-    try:
-        if (path / 'qmk.json').is_file():
-            userspace_defs = UserspaceDefs(path / 'qmk.json')
-            if userspace_defs is not None:
-                return True
-    except jsonschema.ValidationError as err:
-        if validation_error_callback is not None:
-            validation_error_callback(err)
-        pass
-    return False
+def qmk_userspace_validate(path):
+    if (path / 'qmk.json').is_file():
+        UserspaceDefs(path / 'qmk.json')
 
 
 def detect_qmk_userspace():
     test_dirs = qmk_userspace_paths()
     for test_dir in test_dirs:
-        if qmk_userspace_validate(test_dir):
+        try:
+            qmk_userspace_validate(test_dir)
             return test_dir
+        except UserspaceValidationException:
+            pass
     return None
 
 
@@ -50,15 +45,26 @@ class UserspaceDefs:
     def __init__(self, userspace_json: Path):
         self.path = userspace_json
         json = json_load(userspace_json)
-        validate(json, 'qmk.user_repo.v0')  # `qmk.json` must have a userspace_version at minimum
 
-        # TODO: validate different versions of json when we're ready to deal with versioning
-        # Start at highest version, then run down the list to v1
+        exception = UserspaceValidationException()
+        success = False
+
+        try:
+            validate(json, 'qmk.user_repo.v0')  # `qmk.json` must have a userspace_version at minimum
+        except jsonschema.ValidationError as err:
+            exception.add('qmk.user_repo.v0', err)
+            raise exception
+
+        # Iterate through each version of the schema, starting with the latest and decreasing to v1
         try:
             validate(json, 'qmk.user_repo.v1')
             self.__load_v1(json)
-        except jsonschema.ValidationError:
-            raise  # v1 always needs to raise here -- higher versions shouldn't bother and should just `pass`
+            success = True
+        except jsonschema.ValidationError as err:
+            exception.add('qmk.user_repo.v1', err)
+
+        if not success:
+            raise exception
 
     def save(self):
         target_json = {
@@ -77,3 +83,21 @@ def save(self):
 
     def __load_v1(self, json):
         self.build_targets = json['build_targets']
+
+
+class UserspaceValidationException(Exception):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__exceptions = []
+
+    def __str__(self):
+        return self.message
+
+    @property
+    def exceptions(self):
+        return self.__exceptions
+
+    def add(self, schema, exception):
+        self.__exceptions.append((schema, exception))
+        errorlist = "\n\n".join([f"{schema}: {exception}" for schema, exception in self.__exceptions])
+        self.message = f'Could not validate against any version of the userspace schema. Errors:\n\n{errorlist}'
\ No newline at end of file

From 01e3808c7f062b9fa98cac0a496a727b61f59de2 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 22 Sep 2023 21:00:45 +1000
Subject: [PATCH 23/84] `qmk userspace-(add|remove|list)`

---
 data/schemas/user_repo_v1.jsonschema   |  2 +-
 lib/python/qmk/cli/userspace/add.py    |  8 +++++++-
 lib/python/qmk/cli/userspace/list.py   |  6 +++++-
 lib/python/qmk/cli/userspace/remove.py |  8 +++++++-
 lib/python/qmk/userspace.py            | 23 ++++++++++++++++++++---
 5 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/data/schemas/user_repo_v1.jsonschema b/data/schemas/user_repo_v1.jsonschema
index 751347365675..6a2efc2e9521 100644
--- a/data/schemas/user_repo_v1.jsonschema
+++ b/data/schemas/user_repo_v1.jsonschema
@@ -21,7 +21,7 @@
                 "items": {
                     "type": "string",
                     "minLength": 1,
-                    "pattern": "^[0-9a-z][0-9a-z_]*$"
+                    "pattern": "^[0-9a-z][0-9a-z_/]*$"
                 }
             }
         }
diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index e0f1de4bfbd1..d5b36d65046a 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -1,13 +1,19 @@
 from milc import cli
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
+from qmk.keymap import keymap_completer
 from qmk.userspace import UserspaceDefs
 
 
+@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.subcommand('Adds a build target to userspace `qmk.json`.')
 def userspace_add(cli):
     if not HAS_QMK_USERSPACE:
         cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
         return False
 
-    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
\ No newline at end of file
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+    userspace.add_target(cli.args.keyboard, cli.args.keymap)
+    userspace.save()
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index 9c0d0b705f6e..1b94b9d91f09 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -10,4 +10,8 @@ def userspace_list(cli):
         cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
         return False
 
-    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
\ No newline at end of file
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+    for target in userspace.build_targets:
+        keyboard = target['keyboard']
+        keymap = target['keymap']
+        cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
\ No newline at end of file
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
index aa244d800eb2..d4b3e6387442 100644
--- a/lib/python/qmk/cli/userspace/remove.py
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -1,13 +1,19 @@
 from milc import cli
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
+from qmk.keymap import keymap_completer
 from qmk.userspace import UserspaceDefs
 
 
+@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.subcommand('Removes a build target from userspace `qmk.json`.')
 def userspace_remove(cli):
     if not HAS_QMK_USERSPACE:
         cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
         return False
 
-    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
\ No newline at end of file
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+    userspace.remove_target(cli.args.keyboard, cli.args.keymap)
+    userspace.save()
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index 8d050b2ca078..e7f4add1d03c 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -44,6 +44,7 @@ def detect_qmk_userspace():
 class UserspaceDefs:
     def __init__(self, userspace_json: Path):
         self.path = userspace_json
+        self.build_targets = []
         json = json_load(userspace_json)
 
         exception = UserspaceValidationException()
@@ -69,20 +70,36 @@ def __init__(self, userspace_json: Path):
     def save(self):
         target_json = {
             "userspace_version": "1.0",  # Needs to match latest version
-            "build_targets":
-                self.build_targets  # Only other field needed in v1
+            "build_targets": []
         }
 
+        for e in self.build_targets:
+            target_json['build_targets'].append([e['keyboard'], e['keymap']])
+
         try:
             # Ensure what we're writing validates against the latest version of the schema
             validate(target_json, 'qmk.user_repo.v1')
         except jsonschema.ValidationError as err:
             cli.log.error(f'Could not save userspace file: {err}')
+            return False
 
         self.path.write_text(json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True))
+        return True
+
+    def add_target(self, keyboard, keymap):
+        e = {"keyboard": keyboard, "keymap": keymap}
+        if e not in self.build_targets:
+            self.build_targets.append(e)
+
+    def remove_target(self, keyboard, keymap):
+        e = {"keyboard": keyboard, "keymap": keymap}
+        if e in self.build_targets:
+            self.build_targets.remove(e)
 
     def __load_v1(self, json):
-        self.build_targets = json['build_targets']
+        for e in json['build_targets']:
+            if len(e) == 2:
+                self.add_target(e[0], e[1])
 
 
 class UserspaceValidationException(Exception):

From 9e9c09ff5a4a07e6a8bd0dd93168825f62309c28 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 22 Sep 2023 21:14:29 +1000
Subject: [PATCH 24/84] Try to determine userspace from qmk config, if set.

---
 Makefile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Makefile b/Makefile
index d5404286b1a1..25209d21df88 100644
--- a/Makefile
+++ b/Makefile
@@ -38,6 +38,11 @@ $(info QMK Firmware $(QMK_VERSION))
 endif
 endif
 
+# Try to determine userspace from qmk config, if set.
+ifeq ($(QMK_USERSPACE),)
+    QMK_USERSPACE = $(shell qmk config -ro user.overlay_dir | cut -d= -f2 | sed -e 's@^None$$@@g')
+endif
+
 # Determine which qmk cli to use
 QMK_BIN := qmk
 

From 0e807a567fe916c2b82b6e467de3cb45b3f9357a Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 22 Sep 2023 22:03:07 +1000
Subject: [PATCH 25/84] Validate keymap targets.

---
 lib/python/qmk/cli/compile.py       | 14 ++------------
 lib/python/qmk/cli/flash.py         | 14 ++------------
 lib/python/qmk/cli/userspace/add.py |  6 +++++-
 lib/python/qmk/keymap.py            | 10 ++++++++++
 4 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 66126dd2a3c5..52002121eca5 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -10,17 +10,7 @@
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, create_make_target, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
-from qmk.keymap import keymap_completer, locate_keymap
-
-
-def _is_keymap_target(keyboard, keymap):
-    if keymap == 'all':
-        return True
-
-    if locate_keymap(keyboard, keymap):
-        return True
-
-    return False
+from qmk.keymap import keymap_completer, is_keymap_target
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
@@ -61,7 +51,7 @@ def compile(cli):
 
     elif cli.config.compile.keyboard and cli.config.compile.keymap:
         # Generate the make command for a specific keyboard/keymap.
-        if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
+        if not is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
             cli.log.error('Invalid keymap argument.')
             cli.print_help()
             return False
diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
index 8724f26889b2..73dea40e95a9 100644
--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -11,20 +11,10 @@
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.keymap import keymap_completer, locate_keymap
+from qmk.keymap import keymap_completer, is_keymap_target
 from qmk.flashers import flasher
 
 
-def _is_keymap_target(keyboard, keymap):
-    if keymap == 'all':
-        return True
-
-    if locate_keymap(keyboard, keymap):
-        return True
-
-    return False
-
-
 def _list_bootloaders():
     """Prints the available bootloaders listed in docs.qmk.fm.
     """
@@ -108,7 +98,7 @@ def flash(cli):
 
     elif cli.config.flash.keyboard and cli.config.flash.keymap:
         # Generate the make command for a specific keyboard/keymap.
-        if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
+        if not is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
             cli.log.error('Invalid keymap argument.')
             cli.print_help()
             return False
diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index d5b36d65046a..b440223303d1 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -2,7 +2,7 @@
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
-from qmk.keymap import keymap_completer
+from qmk.keymap import keymap_completer, is_keymap_target
 from qmk.userspace import UserspaceDefs
 
 
@@ -14,6 +14,10 @@ def userspace_add(cli):
         cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
         return False
 
+    if not is_keymap_target(cli.args.keyboard, cli.args.keymap):
+        cli.log.error('Invalid keymap argument.')
+        return False
+
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
     userspace.add_target(cli.args.keyboard, cli.args.keymap)
     userspace.save()
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index 340425740f2d..8f52ea30a61a 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -465,6 +465,16 @@ def locate_keymap(keyboard, keymap):
                     return community_layout / 'keymap.c'
 
 
+def is_keymap_target(keyboard, keymap):
+    if keymap == 'all':
+        return True
+
+    if locate_keymap(keyboard, keymap):
+        return True
+
+    return False
+
+
 def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False):
     """List the available keymaps for a keyboard.
 

From 83a5b6061579e805dfdd1f3317796e5612045725 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 23 Sep 2023 12:11:07 +1000
Subject: [PATCH 26/84] Validate targets for `qmk userspace-list`

---
 lib/python/qmk/cli/userspace/list.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index 1b94b9d91f09..7a98e5af23ab 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -2,6 +2,8 @@
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.userspace import UserspaceDefs
+from qmk.keyboard import keyboard_folder_or_all, is_all_keyboards
+from qmk.keymap import is_keymap_target
 
 
 @cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
@@ -12,6 +14,9 @@ def userspace_list(cli):
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
     for target in userspace.build_targets:
-        keyboard = target['keyboard']
+        keyboard = keyboard_folder_or_all(target['keyboard'])
         keymap = target['keymap']
-        cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
\ No newline at end of file
+        if is_all_keyboards(keyboard) or is_keymap_target(keyboard, keymap):
+            cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
+        else:
+            cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!')

From 42a57611741e7fa0785679c094c340e877a75174 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 23 Sep 2023 12:11:59 +1000
Subject: [PATCH 27/84] `qmk userspace-compile` for consistency.

---
 lib/python/qmk/cli/__init__.py                        | 2 +-
 lib/python/qmk/cli/userspace/__init__.py              | 2 +-
 lib/python/qmk/cli/userspace/{build.py => compile.py} | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)
 rename lib/python/qmk/cli/userspace/{build.py => compile.py} (76%)

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 9e4c2ec89032..486efc9d3927 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -80,7 +80,7 @@
     'qmk.cli.painter',
     'qmk.cli.pytest',
     'qmk.cli.userspace.add',
-    'qmk.cli.userspace.build',
+    'qmk.cli.userspace.compile',
     'qmk.cli.userspace.doctor',
     'qmk.cli.userspace.list',
     'qmk.cli.userspace.remove',
diff --git a/lib/python/qmk/cli/userspace/__init__.py b/lib/python/qmk/cli/userspace/__init__.py
index c2ac6576aa8c..5757d3a4c994 100644
--- a/lib/python/qmk/cli/userspace/__init__.py
+++ b/lib/python/qmk/cli/userspace/__init__.py
@@ -2,4 +2,4 @@
 from . import add
 from . import remove
 from . import list
-from . import build
+from . import compile
diff --git a/lib/python/qmk/cli/userspace/build.py b/lib/python/qmk/cli/userspace/compile.py
similarity index 76%
rename from lib/python/qmk/cli/userspace/build.py
rename to lib/python/qmk/cli/userspace/compile.py
index 373a13256ba2..5769e031bb40 100644
--- a/lib/python/qmk/cli/userspace/build.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -4,8 +4,8 @@
 from qmk.userspace import UserspaceDefs
 
 
-@cli.subcommand('Builds the build targets specified in userspace `qmk.json`.')
-def userspace_build(cli):
+@cli.subcommand('Compiles the build targets specified in userspace `qmk.json`.')
+def userspace_compile(cli):
     if not HAS_QMK_USERSPACE:
         cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
         return False

From 0df23179fa02c612dc8b06b75a8098a0d141d562 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 23 Sep 2023 14:03:08 +1000
Subject: [PATCH 28/84] Allow for `qmk mass-compile all:<keymap>`

---
 lib/python/qmk/cli/mass_compile.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 5e0c784892ae..44193572416e 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -42,7 +42,17 @@ def mass_compile(cli):
     makefile = builddir / 'parallel_kb_builds.mk'
 
     if len(cli.args.builds) > 0:
-        targets = list(sorted(set([(resolve_keyboard(e[0]), e[1]) for e in [b.split(':') for b in cli.args.builds]])))
+        targets = []
+        for target in cli.args.builds:
+            split_target = target.split(':')
+            if len(split_target) != 2:
+                cli.log.error(f"Invalid build target: {target}")
+                return False
+            if split_target[0] == 'all':
+                targets.extend(search_keymap_targets(split_target[1], []))
+            else:
+                targets.append((resolve_keyboard(split_target[0]), split_target[1]))
+        targets = list(sorted(set([(e[0], e[1]) for e in targets])))
     else:
         targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
 

From 317cfed4d18043c260cf94695a6a2e8b7add9325 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 10:13:54 +1000
Subject: [PATCH 29/84] Allow for disabling of parallel processing of `qmk
 find` and `qmk mass-compile`.

---
 lib/python/qmk/cli/find.py         |  3 +-
 lib/python/qmk/cli/mass_compile.py |  5 ++--
 lib/python/qmk/search.py           | 46 ++++++++++++++++++++----------
 3 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index 2836eb8a5421..b11f35fce5a3 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -15,6 +15,7 @@
 )
 @cli.argument('-p', '--print', arg_only=True, action='append', default=[], help="For each matched target, print the value of the supplied info.json key. May be passed multiple times.")
 @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
+@cli.argument('-x', '--disable-parallel-parsing', arg_only=True, action='store_true', help="Disables parallel parsing of files, useful for debugging stalls.")
 @cli.subcommand('Find builds which match supplied search criteria.')
 def find(cli):
     """Search through all keyboards and keymaps for a given search criteria.
@@ -23,7 +24,7 @@ def find(cli):
     if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
         cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
 
-    targets = search_keymap_targets(cli.args.keymap, cli.args.filter, cli.args.print)
+    targets = search_keymap_targets(cli.args.keymap, cli.args.filter, cli.args.print, parallel=not cli.args.disable_parallel_parsing)
     for keyboard, keymap, print_vals in targets:
         print(f'{keyboard}:{keymap}')
 
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index a758e1994872..b12ad8fdf256 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -86,6 +86,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
+@cli.argument('-x', '--disable-parallel-parsing', arg_only=True, action='store_true', help="Disables parallel parsing of files, useful for debugging stalls.")
 @cli.argument(
     '-f',
     '--filter',
@@ -102,8 +103,8 @@ def mass_compile(cli):
     """Compile QMK Firmware against all keyboards.
     """
     if len(cli.args.builds) > 0:
-        targets = search_make_targets(cli.args.builds, cli.args.filter)
+        targets = search_make_targets(cli.args.builds, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
     else:
-        targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
+        targets = search_keymap_targets(cli.args.keymap, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
 
     return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.args.parallel, cli.args.env)
diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 0b5d48921898..5da083a45562 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -54,7 +54,7 @@ def _load_keymap_info(kb_km):
         return (kb_km[0], kb_km[1], keymap_json(kb_km[0], kb_km[1]))
 
 
-def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]:
+def expand_make_targets(targets: List[str], parallel=True) -> List[Tuple[str, str]]:
     """Expand a list of make targets into a list of (keyboard, keymap) tuples.
 
     Caters for 'all' in either keyboard or keymap, or both.
@@ -66,10 +66,10 @@ def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]:
             cli.log.error(f"Invalid build target: {target}")
             return []
         split_targets.append((split_target[0], split_target[1]))
-    return expand_keymap_targets(split_targets)
+    return expand_keymap_targets(split_targets, parallel)
 
 
-def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = None) -> List[Tuple[str, str]]:
+def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = None, parallel=True) -> List[Tuple[str, str]]:
     """Expand a keyboard input and keymap input into a list of (keyboard, keymap) tuples.
 
     Caters for 'all' in either keyboard or keymap, or both.
@@ -78,17 +78,25 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
         all_keyboards = qmk.keyboard.list_keyboards()
 
     if keyboard == 'all':
-        with multiprocessing.Pool() as pool:
+
+        def _inner_func(pool):
+            _map_func = pool.imap_unordered if pool is not None else map
             if keymap == 'all':
                 cli.log.info('Retrieving list of all keyboards and keymaps...')
                 targets = []
-                for kb in pool.imap_unordered(_all_keymaps, all_keyboards):
+                for kb in _map_func(_all_keymaps, all_keyboards):
                     targets.extend(kb)
                 return targets
             else:
                 cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...')
                 keyboard_filter = functools.partial(_keymap_exists, keymap=keymap)
-                return [(kb, keymap) for kb in filter(lambda e: e is not None, pool.imap_unordered(keyboard_filter, all_keyboards))]
+                return [(kb, keymap) for kb in filter(lambda e: e is not None, _map_func(keyboard_filter, all_keyboards))]
+
+        if parallel:
+            with multiprocessing.Pool() as pool:
+                return _inner_func(pool)
+        return _inner_func(None)
+
     else:
         if keymap == 'all':
             keyboard = qmk.keyboard.resolve_keyboard(keyboard)
@@ -98,17 +106,17 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
             return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)]
 
 
-def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+def expand_keymap_targets(targets: List[Tuple[str, str]], parallel=True) -> List[Tuple[str, str]]:
     """Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
     """
     overall_targets = []
     all_keyboards = qmk.keyboard.list_keyboards()
     for target in targets:
-        overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
+        overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards, parallel))
     return list(sorted(set(overall_targets)))
 
 
-def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
 
     Optionally includes the values of the queried info.json keys.
@@ -117,8 +125,16 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
         targets = [(kb, km, {}) for kb, km in target_list]
     else:
         cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
-        with multiprocessing.Pool() as pool:
-            valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in pool.imap_unordered(_load_keymap_info, target_list)]
+
+        def _inner_func(pool):
+            _map_func = pool.imap_unordered if pool is not None else map
+            return [(e[0], e[1], dotty(e[2])) for e in _map_func(_load_keymap_info, target_list)]
+
+        if parallel:
+            with multiprocessing.Pool() as pool:
+                valid_keymaps = _inner_func(pool)
+        else:
+            valid_keymaps = _inner_func(None)
 
         function_re = re.compile(r'^(?P<function>[a-zA-Z]+)\((?P<key>[a-zA-Z0-9_\.]+)(,\s*(?P<value>[^#]+))?\)$')
         equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
@@ -179,13 +195,13 @@ def f(e):
     return targets
 
 
-def search_keymap_targets(keymap='default', filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_keymap_targets(keymap='default', filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_keymap_targets([('all', keymap)]), filters, print_vals), key=lambda e: (e[0], e[1])))
+    return list(sorted(_filter_keymap_targets(expand_keymap_targets([('all', keymap)], parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))
 
 
-def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
+    return list(sorted(_filter_keymap_targets(expand_make_targets(targets, parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))

From 260dee66a86e96e49a5e0bd6d199e7643536c6c3 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 10:39:53 +1000
Subject: [PATCH 30/84] `qmk userspace-compile`

---
 lib/python/qmk/cli/find.py              |  2 +-
 lib/python/qmk/cli/mass_compile.py      |  4 ++--
 lib/python/qmk/cli/userspace/compile.py | 11 ++++++++++-
 lib/python/qmk/search.py                |  4 ++--
 lib/python/qmk/userspace.py             | 16 ++++++++++------
 5 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index b11f35fce5a3..364a57886b63 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -24,7 +24,7 @@ def find(cli):
     if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
         cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
 
-    targets = search_keymap_targets(cli.args.keymap, cli.args.filter, cli.args.print, parallel=not cli.args.disable_parallel_parsing)
+    targets = search_keymap_targets([('all', cli.args.keymap)], cli.args.filter, cli.args.print, parallel=not cli.args.disable_parallel_parsing)
     for keyboard, keymap, print_vals in targets:
         print(f'{keyboard}:{keymap}')
 
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index b12ad8fdf256..1e95f1d72602 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -32,7 +32,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
         builddir.mkdir(parents=True, exist_ok=True)
         with open(makefile, "w") as f:
 
-            envs = build_environment(cli.args.env)
+            envs = build_environment(env)
 
             for target in sorted(targets):
                 keyboard_name = target[0]
@@ -105,6 +105,6 @@ def mass_compile(cli):
     if len(cli.args.builds) > 0:
         targets = search_make_targets(cli.args.builds, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
     else:
-        targets = search_keymap_targets(cli.args.keymap, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
+        targets = search_keymap_targets([('all', cli.args.keymap)], cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
 
     return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.args.parallel, cli.args.env)
diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
index 5769e031bb40..bedadafe3380 100644
--- a/lib/python/qmk/cli/userspace/compile.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -2,12 +2,21 @@
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.userspace import UserspaceDefs
+from qmk.search import search_keymap_targets
+from qmk.cli.mass_compile import mass_compile_targets
 
 
+@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
+@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
+@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
+@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
+@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
 @cli.subcommand('Compiles the build targets specified in userspace `qmk.json`.')
 def userspace_compile(cli):
     if not HAS_QMK_USERSPACE:
         cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
         return False
 
-    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
\ No newline at end of file
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+    make_targets = search_keymap_targets(userspace.targets)
+    mass_compile_targets(make_targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.args.parallel, cli.args.env)
diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 5da083a45562..07ec1aa6f651 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -195,10 +195,10 @@ def f(e):
     return targets
 
 
-def search_keymap_targets(keymap='default', filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_keymap_targets([('all', keymap)], parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))
+    return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets, parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))
 
 
 def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index e7f4add1d03c..abe4f8acf9ed 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -44,7 +44,7 @@ def detect_qmk_userspace():
 class UserspaceDefs:
     def __init__(self, userspace_json: Path):
         self.path = userspace_json
-        self.build_targets = []
+        self._build_targets = []
         json = json_load(userspace_json)
 
         exception = UserspaceValidationException()
@@ -73,7 +73,7 @@ def save(self):
             "build_targets": []
         }
 
-        for e in self.build_targets:
+        for e in self._build_targets:
             target_json['build_targets'].append([e['keyboard'], e['keymap']])
 
         try:
@@ -86,15 +86,19 @@ def save(self):
         self.path.write_text(json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True))
         return True
 
+    @property
+    def targets(self):
+        return [(e['keyboard'], e['keymap']) for e in self._build_targets]
+
     def add_target(self, keyboard, keymap):
         e = {"keyboard": keyboard, "keymap": keymap}
-        if e not in self.build_targets:
-            self.build_targets.append(e)
+        if e not in self._build_targets:
+            self._build_targets.append(e)
 
     def remove_target(self, keyboard, keymap):
         e = {"keyboard": keyboard, "keymap": keymap}
-        if e in self.build_targets:
-            self.build_targets.remove(e)
+        if e in self._build_targets:
+            self._build_targets.remove(e)
 
     def __load_v1(self, json):
         for e in json['build_targets']:

From 6c0740bb8c3f834149041da130b839c9cbda2339 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 10:13:54 +1000
Subject: [PATCH 31/84] Allow for disabling of parallel processing of `qmk
 find` and `qmk mass-compile`.

---
 lib/python/qmk/cli/find.py         |  3 +-
 lib/python/qmk/cli/mass_compile.py |  5 ++--
 lib/python/qmk/search.py           | 46 ++++++++++++++++++++----------
 3 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index 2836eb8a5421..b11f35fce5a3 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -15,6 +15,7 @@
 )
 @cli.argument('-p', '--print', arg_only=True, action='append', default=[], help="For each matched target, print the value of the supplied info.json key. May be passed multiple times.")
 @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
+@cli.argument('-x', '--disable-parallel-parsing', arg_only=True, action='store_true', help="Disables parallel parsing of files, useful for debugging stalls.")
 @cli.subcommand('Find builds which match supplied search criteria.')
 def find(cli):
     """Search through all keyboards and keymaps for a given search criteria.
@@ -23,7 +24,7 @@ def find(cli):
     if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
         cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
 
-    targets = search_keymap_targets(cli.args.keymap, cli.args.filter, cli.args.print)
+    targets = search_keymap_targets(cli.args.keymap, cli.args.filter, cli.args.print, parallel=not cli.args.disable_parallel_parsing)
     for keyboard, keymap, print_vals in targets:
         print(f'{keyboard}:{keymap}')
 
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 1227f435e74e..16ee64515e41 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -79,6 +79,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
+@cli.argument('-x', '--disable-parallel-parsing', arg_only=True, action='store_true', help="Disables parallel parsing of files, useful for debugging stalls.")
 @cli.argument(
     '-f',
     '--filter',
@@ -95,8 +96,8 @@ def mass_compile(cli):
     """Compile QMK Firmware against all keyboards.
     """
     if len(cli.args.builds) > 0:
-        targets = search_make_targets(cli.args.builds, cli.args.filter)
+        targets = search_make_targets(cli.args.builds, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
     else:
-        targets = search_keymap_targets(cli.args.keymap, cli.args.filter)
+        targets = search_keymap_targets(cli.args.keymap, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
 
     return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.args.parallel, cli.args.env)
diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 0b5d48921898..5da083a45562 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -54,7 +54,7 @@ def _load_keymap_info(kb_km):
         return (kb_km[0], kb_km[1], keymap_json(kb_km[0], kb_km[1]))
 
 
-def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]:
+def expand_make_targets(targets: List[str], parallel=True) -> List[Tuple[str, str]]:
     """Expand a list of make targets into a list of (keyboard, keymap) tuples.
 
     Caters for 'all' in either keyboard or keymap, or both.
@@ -66,10 +66,10 @@ def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]:
             cli.log.error(f"Invalid build target: {target}")
             return []
         split_targets.append((split_target[0], split_target[1]))
-    return expand_keymap_targets(split_targets)
+    return expand_keymap_targets(split_targets, parallel)
 
 
-def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = None) -> List[Tuple[str, str]]:
+def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = None, parallel=True) -> List[Tuple[str, str]]:
     """Expand a keyboard input and keymap input into a list of (keyboard, keymap) tuples.
 
     Caters for 'all' in either keyboard or keymap, or both.
@@ -78,17 +78,25 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
         all_keyboards = qmk.keyboard.list_keyboards()
 
     if keyboard == 'all':
-        with multiprocessing.Pool() as pool:
+
+        def _inner_func(pool):
+            _map_func = pool.imap_unordered if pool is not None else map
             if keymap == 'all':
                 cli.log.info('Retrieving list of all keyboards and keymaps...')
                 targets = []
-                for kb in pool.imap_unordered(_all_keymaps, all_keyboards):
+                for kb in _map_func(_all_keymaps, all_keyboards):
                     targets.extend(kb)
                 return targets
             else:
                 cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...')
                 keyboard_filter = functools.partial(_keymap_exists, keymap=keymap)
-                return [(kb, keymap) for kb in filter(lambda e: e is not None, pool.imap_unordered(keyboard_filter, all_keyboards))]
+                return [(kb, keymap) for kb in filter(lambda e: e is not None, _map_func(keyboard_filter, all_keyboards))]
+
+        if parallel:
+            with multiprocessing.Pool() as pool:
+                return _inner_func(pool)
+        return _inner_func(None)
+
     else:
         if keymap == 'all':
             keyboard = qmk.keyboard.resolve_keyboard(keyboard)
@@ -98,17 +106,17 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
             return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)]
 
 
-def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+def expand_keymap_targets(targets: List[Tuple[str, str]], parallel=True) -> List[Tuple[str, str]]:
     """Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
     """
     overall_targets = []
     all_keyboards = qmk.keyboard.list_keyboards()
     for target in targets:
-        overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
+        overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards, parallel))
     return list(sorted(set(overall_targets)))
 
 
-def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
 
     Optionally includes the values of the queried info.json keys.
@@ -117,8 +125,16 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
         targets = [(kb, km, {}) for kb, km in target_list]
     else:
         cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
-        with multiprocessing.Pool() as pool:
-            valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in pool.imap_unordered(_load_keymap_info, target_list)]
+
+        def _inner_func(pool):
+            _map_func = pool.imap_unordered if pool is not None else map
+            return [(e[0], e[1], dotty(e[2])) for e in _map_func(_load_keymap_info, target_list)]
+
+        if parallel:
+            with multiprocessing.Pool() as pool:
+                valid_keymaps = _inner_func(pool)
+        else:
+            valid_keymaps = _inner_func(None)
 
         function_re = re.compile(r'^(?P<function>[a-zA-Z]+)\((?P<key>[a-zA-Z0-9_\.]+)(,\s*(?P<value>[^#]+))?\)$')
         equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
@@ -179,13 +195,13 @@ def f(e):
     return targets
 
 
-def search_keymap_targets(keymap='default', filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_keymap_targets(keymap='default', filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_keymap_targets([('all', keymap)]), filters, print_vals), key=lambda e: (e[0], e[1])))
+    return list(sorted(_filter_keymap_targets(expand_keymap_targets([('all', keymap)], parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))
 
 
-def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
+    return list(sorted(_filter_keymap_targets(expand_make_targets(targets, parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))

From 2808171cad9f9eff317b2a95dcc92ecd168a1f75 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 10:39:53 +1000
Subject: [PATCH 32/84] Parameterise `search_keymap_targets()` so that it can
 be fed an array of tuples, instead of just a keymap name.

---
 lib/python/qmk/cli/find.py         | 2 +-
 lib/python/qmk/cli/mass_compile.py | 2 +-
 lib/python/qmk/search.py           | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index b11f35fce5a3..364a57886b63 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -24,7 +24,7 @@ def find(cli):
     if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
         cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
 
-    targets = search_keymap_targets(cli.args.keymap, cli.args.filter, cli.args.print, parallel=not cli.args.disable_parallel_parsing)
+    targets = search_keymap_targets([('all', cli.args.keymap)], cli.args.filter, cli.args.print, parallel=not cli.args.disable_parallel_parsing)
     for keyboard, keymap, print_vals in targets:
         print(f'{keyboard}:{keymap}')
 
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 16ee64515e41..ccf4c53408f7 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -98,6 +98,6 @@ def mass_compile(cli):
     if len(cli.args.builds) > 0:
         targets = search_make_targets(cli.args.builds, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
     else:
-        targets = search_keymap_targets(cli.args.keymap, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
+        targets = search_keymap_targets([('all', cli.args.keymap)], cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
 
     return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.args.parallel, cli.args.env)
diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 5da083a45562..07ec1aa6f651 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -195,10 +195,10 @@ def f(e):
     return targets
 
 
-def search_keymap_targets(keymap='default', filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_keymap_targets([('all', keymap)], parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))
+    return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets, parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))
 
 
 def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:

From 06316a1cd44c84740b2277e8938c4d39bf2ef35c Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 11:25:53 +1000
Subject: [PATCH 33/84] Simplification of parallelisation.

---
 lib/python/qmk/search.py | 66 ++++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 29 deletions(-)

diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 07ec1aa6f651..b31f615f4f12 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -25,6 +25,24 @@ def _set_log_level(level):
     return old
 
 
+@contextlib.contextmanager
+def parallelize(parallel):
+    with contextlib.suppress(ImportError):
+        from mpire import WorkerPool
+        if parallel:
+            with WorkerPool() as pool:
+                yield functools.partial(pool.imap_unordered, progress_bar=True)
+        else:
+            yield map
+        return
+
+    if parallel:
+        with multiprocessing.Pool() as pool:
+            yield pool.imap_unordered
+    else:
+        yield map
+
+
 @contextlib.contextmanager
 def ignore_logging():
     old = _set_log_level(logging.CRITICAL)
@@ -47,11 +65,15 @@ def _keymap_exists(keyboard, keymap):
         return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
 
 
-def _load_keymap_info(kb_km):
+def _load_keymap_info(arg0, arg1=None):
     """Returns a tuple of (keyboard, keymap, info.json) for the given keyboard/keymap combination.
+
+    Caters for the different unpacking requirements of each variatn of imap_unordered().
     """
     with ignore_logging():
-        return (kb_km[0], kb_km[1], keymap_json(kb_km[0], kb_km[1]))
+        if arg1 is None:
+            return (arg0[0], arg0[1], keymap_json(arg0[0], arg0[1]))
+        return (arg0, arg1, keymap_json(arg0, arg1))
 
 
 def expand_make_targets(targets: List[str], parallel=True) -> List[Tuple[str, str]]:
@@ -78,25 +100,18 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
         all_keyboards = qmk.keyboard.list_keyboards()
 
     if keyboard == 'all':
-
-        def _inner_func(pool):
-            _map_func = pool.imap_unordered if pool is not None else map
-            if keymap == 'all':
-                cli.log.info('Retrieving list of all keyboards and keymaps...')
-                targets = []
-                for kb in _map_func(_all_keymaps, all_keyboards):
+        if keymap == 'all':
+            cli.log.info('Retrieving list of all keyboards and keymaps...')
+            targets = []
+            with parallelize(parallel) as map_func:
+                for kb in map_func(_all_keymaps, all_keyboards):
                     targets.extend(kb)
-                return targets
-            else:
-                cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...')
-                keyboard_filter = functools.partial(_keymap_exists, keymap=keymap)
-                return [(kb, keymap) for kb in filter(lambda e: e is not None, _map_func(keyboard_filter, all_keyboards))]
-
-        if parallel:
-            with multiprocessing.Pool() as pool:
-                return _inner_func(pool)
-        return _inner_func(None)
-
+            return targets
+        else:
+            cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...')
+            keyboard_filter = functools.partial(_keymap_exists, keymap=keymap)
+            with parallelize(parallel) as map_func:
+                return [(kb, keymap) for kb in filter(lambda e: e is not None, map_func(keyboard_filter, all_keyboards))]
     else:
         if keymap == 'all':
             keyboard = qmk.keyboard.resolve_keyboard(keyboard)
@@ -126,15 +141,8 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
     else:
         cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
 
-        def _inner_func(pool):
-            _map_func = pool.imap_unordered if pool is not None else map
-            return [(e[0], e[1], dotty(e[2])) for e in _map_func(_load_keymap_info, target_list)]
-
-        if parallel:
-            with multiprocessing.Pool() as pool:
-                valid_keymaps = _inner_func(pool)
-        else:
-            valid_keymaps = _inner_func(None)
+        with parallelize(parallel) as map_func:
+            valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in map_func(_load_keymap_info, target_list)]
 
         function_re = re.compile(r'^(?P<function>[a-zA-Z]+)\((?P<key>[a-zA-Z0-9_\.]+)(,\s*(?P<value>[^#]+))?\)$')
         equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')

From 68032751800284d13cc2c128031ec51fd4cdec49 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 11:33:45 +1000
Subject: [PATCH 34/84] Typos and cleanup.

---
 lib/python/qmk/search.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index b31f615f4f12..4c61f3328815 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -27,20 +27,19 @@ def _set_log_level(level):
 
 @contextlib.contextmanager
 def parallelize(parallel):
+    if not parallel:
+        yield map
+        return
+
     with contextlib.suppress(ImportError):
         from mpire import WorkerPool
-        if parallel:
-            with WorkerPool() as pool:
-                yield functools.partial(pool.imap_unordered, progress_bar=True)
-        else:
-            yield map
+        with WorkerPool() as pool:
+            yield functools.partial(pool.imap_unordered, progress_bar=True)
         return
 
-    if parallel:
-        with multiprocessing.Pool() as pool:
-            yield pool.imap_unordered
-    else:
-        yield map
+    with multiprocessing.Pool() as pool:
+        yield pool.imap_unordered
+    return
 
 
 @contextlib.contextmanager
@@ -68,7 +67,7 @@ def _keymap_exists(keyboard, keymap):
 def _load_keymap_info(arg0, arg1=None):
     """Returns a tuple of (keyboard, keymap, info.json) for the given keyboard/keymap combination.
 
-    Caters for the different unpacking requirements of each variatn of imap_unordered().
+    Caters for the different unpacking requirements of each variant of map()/imap_unordered().
     """
     with ignore_logging():
         if arg1 is None:
@@ -140,7 +139,6 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
         targets = [(kb, km, {}) for kb, km in target_list]
     else:
         cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
-
         with parallelize(parallel) as map_func:
             valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in map_func(_load_keymap_info, target_list)]
 

From 0110a4f79de198161187d9a428877fb71a653824 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 12:35:12 +1000
Subject: [PATCH 35/84] Move `parallelize` to util.py.

---
 lib/python/qmk/search.py | 19 +------------------
 lib/python/qmk/util.py   | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 18 deletions(-)
 create mode 100644 lib/python/qmk/util.py

diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 4c61f3328815..9acc59668f6a 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -4,12 +4,12 @@
 import functools
 import fnmatch
 import logging
-import multiprocessing
 import re
 from typing import List, Tuple
 from dotty_dict import dotty
 from milc import cli
 
+from qmk.util import parallelize
 from qmk.info import keymap_json
 import qmk.keyboard
 import qmk.keymap
@@ -25,23 +25,6 @@ def _set_log_level(level):
     return old
 
 
-@contextlib.contextmanager
-def parallelize(parallel):
-    if not parallel:
-        yield map
-        return
-
-    with contextlib.suppress(ImportError):
-        from mpire import WorkerPool
-        with WorkerPool() as pool:
-            yield functools.partial(pool.imap_unordered, progress_bar=True)
-        return
-
-    with multiprocessing.Pool() as pool:
-        yield pool.imap_unordered
-    return
-
-
 @contextlib.contextmanager
 def ignore_logging():
     old = _set_log_level(logging.CRITICAL)
diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py
new file mode 100644
index 000000000000..d57bce951c99
--- /dev/null
+++ b/lib/python/qmk/util.py
@@ -0,0 +1,27 @@
+"""Utility functions.
+"""
+import contextlib
+import functools
+import multiprocessing
+
+@contextlib.contextmanager
+def parallelize(do_parallel):
+    """Returns a function that can be used in place of a map() call.
+
+    Attempts to use `mpire`, falling back to `multiprocessing` if it's not
+    available. If parallelization is not requested, returns the original map()
+    function.
+    """
+    if not do_parallel:
+        yield map
+        return
+
+    with contextlib.suppress(ImportError):
+        from mpire import WorkerPool
+        with WorkerPool() as pool:
+            yield functools.partial(pool.imap_unordered, progress_bar=True)
+        return
+
+    with multiprocessing.Pool() as pool:
+        yield pool.imap_unordered
+    return

From bf40eebf73955f404149d7562b3c9f036afe73f8 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 12:48:13 +1000
Subject: [PATCH 36/84] `qmk format-python`

---
 lib/python/qmk/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py
index d57bce951c99..47c43a76f51c 100644
--- a/lib/python/qmk/util.py
+++ b/lib/python/qmk/util.py
@@ -4,6 +4,7 @@
 import functools
 import multiprocessing
 
+
 @contextlib.contextmanager
 def parallelize(do_parallel):
     """Returns a function that can be used in place of a map() call.
@@ -24,4 +25,3 @@ def parallelize(do_parallel):
 
     with multiprocessing.Pool() as pool:
         yield pool.imap_unordered
-    return

From 4a4cb6c44d2afe675ad44d1289667d17373f1477 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 14:49:32 +1000
Subject: [PATCH 37/84] `cli.args` should not be used for "saveable" settings.

---
 lib/python/qmk/cli/mass_compile.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index ccf4c53408f7..33dac16927e6 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -96,8 +96,8 @@ def mass_compile(cli):
     """Compile QMK Firmware against all keyboards.
     """
     if len(cli.args.builds) > 0:
-        targets = search_make_targets(cli.args.builds, cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
+        targets = search_make_targets(cli.args.builds, cli.args.filter, parallel=not cli.config.mass_compile.disable_parallel_parsing)
     else:
-        targets = search_keymap_targets([('all', cli.args.keymap)], cli.args.filter, parallel=not cli.args.disable_parallel_parsing)
+        targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter, parallel=not cli.config.mass_compile.disable_parallel_parsing)
 
-    return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.args.parallel, cli.args.env)
+    return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env)

From 07b4cec385ab42503c7ebab980f5b80ed0d5420a Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 15:10:46 +1000
Subject: [PATCH 38/84] Use config values.

---
 lib/python/qmk/cli/userspace/compile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
index bedadafe3380..9b5795de4fa7 100644
--- a/lib/python/qmk/cli/userspace/compile.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -19,4 +19,4 @@ def userspace_compile(cli):
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
     make_targets = search_keymap_targets(userspace.targets)
-    mass_compile_targets(make_targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.args.parallel, cli.args.env)
+    mass_compile_targets(make_targets, cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, cli.args.env)

From 7a7b647bd1cd00a3107361724c1c4aba50aecc78 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 15:44:54 +1000
Subject: [PATCH 39/84] Fixup `qmk userspace-list`

---
 lib/python/qmk/cli/userspace/compile.py |  2 +-
 lib/python/qmk/cli/userspace/list.py    | 13 ++++++++++---
 lib/python/qmk/userspace.py             | 16 ++++++----------
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
index 9b5795de4fa7..80af66e6b91d 100644
--- a/lib/python/qmk/cli/userspace/compile.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -18,5 +18,5 @@ def userspace_compile(cli):
         return False
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
-    make_targets = search_keymap_targets(userspace.targets)
+    make_targets = search_keymap_targets([(e['keyboard'], e['keymap']) for e in userspace.build_targets])
     mass_compile_targets(make_targets, cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, cli.args.env)
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index 7a98e5af23ab..7ace9140107d 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -4,8 +4,10 @@
 from qmk.userspace import UserspaceDefs
 from qmk.keyboard import keyboard_folder_or_all, is_all_keyboards
 from qmk.keymap import is_keymap_target
+from qmk.search import search_keymap_targets
 
 
+@cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.")
 @cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
 def userspace_list(cli):
     if not HAS_QMK_USERSPACE:
@@ -13,9 +15,14 @@ def userspace_list(cli):
         return False
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
-    for target in userspace.build_targets:
-        keyboard = keyboard_folder_or_all(target['keyboard'])
-        keymap = target['keymap']
+
+    build_targets = [(e['keyboard'], e['keymap']) for e in userspace.build_targets]
+    if cli.args.expand:
+        build_targets = search_keymap_targets(build_targets)
+
+    for target in build_targets:
+        keyboard = keyboard_folder_or_all(target[0])
+        keymap = target[1]
         if is_all_keyboards(keyboard) or is_keymap_target(keyboard, keymap):
             cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
         else:
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index abe4f8acf9ed..e7f4add1d03c 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -44,7 +44,7 @@ def detect_qmk_userspace():
 class UserspaceDefs:
     def __init__(self, userspace_json: Path):
         self.path = userspace_json
-        self._build_targets = []
+        self.build_targets = []
         json = json_load(userspace_json)
 
         exception = UserspaceValidationException()
@@ -73,7 +73,7 @@ def save(self):
             "build_targets": []
         }
 
-        for e in self._build_targets:
+        for e in self.build_targets:
             target_json['build_targets'].append([e['keyboard'], e['keymap']])
 
         try:
@@ -86,19 +86,15 @@ def save(self):
         self.path.write_text(json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True))
         return True
 
-    @property
-    def targets(self):
-        return [(e['keyboard'], e['keymap']) for e in self._build_targets]
-
     def add_target(self, keyboard, keymap):
         e = {"keyboard": keyboard, "keymap": keymap}
-        if e not in self._build_targets:
-            self._build_targets.append(e)
+        if e not in self.build_targets:
+            self.build_targets.append(e)
 
     def remove_target(self, keyboard, keymap):
         e = {"keyboard": keyboard, "keymap": keymap}
-        if e in self._build_targets:
-            self._build_targets.remove(e)
+        if e in self.build_targets:
+            self.build_targets.remove(e)
 
     def __load_v1(self, json):
         for e in json['build_targets']:

From 5f97a8e16587719999561ceb9fe35b9b87fc2738 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 15:49:27 +1000
Subject: [PATCH 40/84] Cleanup.

---
 lib/python/qmk/cli/userspace/compile.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
index 80af66e6b91d..b24a2d7fd6f0 100644
--- a/lib/python/qmk/cli/userspace/compile.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -18,5 +18,8 @@ def userspace_compile(cli):
         return False
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
-    make_targets = search_keymap_targets([(e['keyboard'], e['keymap']) for e in userspace.build_targets])
-    mass_compile_targets(make_targets, cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, cli.args.env)
+
+    build_targets = [(e['keyboard'], e['keymap']) for e in userspace.build_targets]
+    build_targets = search_keymap_targets(build_targets)
+
+    mass_compile_targets(build_targets, cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, cli.args.env)

From c5aaa25b6c23949de266ed158e6a6973c0441cae Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 15:51:51 +1000
Subject: [PATCH 41/84] `qmk config`

---
 lib/python/qmk/cli/find.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index 364a57886b63..b99f04bca63a 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -24,7 +24,7 @@ def find(cli):
     if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
         cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
 
-    targets = search_keymap_targets([('all', cli.args.keymap)], cli.args.filter, cli.args.print, parallel=not cli.args.disable_parallel_parsing)
+    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print, parallel=not cli.config.find.disable_parallel_parsing)
     for keyboard, keymap, print_vals in targets:
         print(f'{keyboard}:{keymap}')
 

From 74d37f8e4e666f27b0dcad84615c90e60f31bf65 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 16:06:38 +1000
Subject: [PATCH 42/84] Further simplification.

---
 lib/python/qmk/cli/find.py         |  3 +--
 lib/python/qmk/cli/mass_compile.py |  5 ++---
 lib/python/qmk/search.py           | 33 ++++++++++++++----------------
 lib/python/qmk/util.py             | 31 ++++++++++++++++++++++++++--
 4 files changed, 47 insertions(+), 25 deletions(-)

diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index b99f04bca63a..f2135bbc16c8 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -15,7 +15,6 @@
 )
 @cli.argument('-p', '--print', arg_only=True, action='append', default=[], help="For each matched target, print the value of the supplied info.json key. May be passed multiple times.")
 @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
-@cli.argument('-x', '--disable-parallel-parsing', arg_only=True, action='store_true', help="Disables parallel parsing of files, useful for debugging stalls.")
 @cli.subcommand('Find builds which match supplied search criteria.')
 def find(cli):
     """Search through all keyboards and keymaps for a given search criteria.
@@ -24,7 +23,7 @@ def find(cli):
     if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
         cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
 
-    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print, parallel=not cli.config.find.disable_parallel_parsing)
+    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
     for keyboard, keymap, print_vals in targets:
         print(f'{keyboard}:{keymap}')
 
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 33dac16927e6..06e6e411a72a 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -79,7 +79,6 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
-@cli.argument('-x', '--disable-parallel-parsing', arg_only=True, action='store_true', help="Disables parallel parsing of files, useful for debugging stalls.")
 @cli.argument(
     '-f',
     '--filter',
@@ -96,8 +95,8 @@ def mass_compile(cli):
     """Compile QMK Firmware against all keyboards.
     """
     if len(cli.args.builds) > 0:
-        targets = search_make_targets(cli.args.builds, cli.args.filter, parallel=not cli.config.mass_compile.disable_parallel_parsing)
+        targets = search_make_targets(cli.args.builds, cli.args.filter)
     else:
-        targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter, parallel=not cli.config.mass_compile.disable_parallel_parsing)
+        targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
 
     return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env)
diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 9acc59668f6a..12b239ac6722 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -9,7 +9,7 @@
 from dotty_dict import dotty
 from milc import cli
 
-from qmk.util import parallelize
+from qmk.util import parallel_map
 from qmk.info import keymap_json
 import qmk.keyboard
 import qmk.keymap
@@ -58,7 +58,7 @@ def _load_keymap_info(arg0, arg1=None):
         return (arg0, arg1, keymap_json(arg0, arg1))
 
 
-def expand_make_targets(targets: List[str], parallel=True) -> List[Tuple[str, str]]:
+def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]:
     """Expand a list of make targets into a list of (keyboard, keymap) tuples.
 
     Caters for 'all' in either keyboard or keymap, or both.
@@ -70,10 +70,10 @@ def expand_make_targets(targets: List[str], parallel=True) -> List[Tuple[str, st
             cli.log.error(f"Invalid build target: {target}")
             return []
         split_targets.append((split_target[0], split_target[1]))
-    return expand_keymap_targets(split_targets, parallel)
+    return expand_keymap_targets(split_targets)
 
 
-def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = None, parallel=True) -> List[Tuple[str, str]]:
+def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = None) -> List[Tuple[str, str]]:
     """Expand a keyboard input and keymap input into a list of (keyboard, keymap) tuples.
 
     Caters for 'all' in either keyboard or keymap, or both.
@@ -85,15 +85,13 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
         if keymap == 'all':
             cli.log.info('Retrieving list of all keyboards and keymaps...')
             targets = []
-            with parallelize(parallel) as map_func:
-                for kb in map_func(_all_keymaps, all_keyboards):
-                    targets.extend(kb)
+            for kb in parallel_map(_all_keymaps, all_keyboards):
+                targets.extend(kb)
             return targets
         else:
             cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...')
             keyboard_filter = functools.partial(_keymap_exists, keymap=keymap)
-            with parallelize(parallel) as map_func:
-                return [(kb, keymap) for kb in filter(lambda e: e is not None, map_func(keyboard_filter, all_keyboards))]
+            return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
     else:
         if keymap == 'all':
             keyboard = qmk.keyboard.resolve_keyboard(keyboard)
@@ -103,17 +101,17 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
             return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)]
 
 
-def expand_keymap_targets(targets: List[Tuple[str, str]], parallel=True) -> List[Tuple[str, str]]:
+def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
     """Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
     """
     overall_targets = []
     all_keyboards = qmk.keyboard.list_keyboards()
     for target in targets:
-        overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards, parallel))
+        overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
     return list(sorted(set(overall_targets)))
 
 
-def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
 
     Optionally includes the values of the queried info.json keys.
@@ -122,8 +120,7 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
         targets = [(kb, km, {}) for kb, km in target_list]
     else:
         cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
-        with parallelize(parallel) as map_func:
-            valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in map_func(_load_keymap_info, target_list)]
+        valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
 
         function_re = re.compile(r'^(?P<function>[a-zA-Z]+)\((?P<key>[a-zA-Z0-9_\.]+)(,\s*(?P<value>[^#]+))?\)$')
         equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
@@ -184,13 +181,13 @@ def f(e):
     return targets
 
 
-def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets, parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))
+    return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
 
 
-def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = [], parallel=True) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_make_targets(targets, parallel), filters, print_vals, parallel), key=lambda e: (e[0], e[1])))
+    return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py
index 47c43a76f51c..87915d2151df 100644
--- a/lib/python/qmk/util.py
+++ b/lib/python/qmk/util.py
@@ -4,24 +4,51 @@
 import functools
 import multiprocessing
 
+from milc import cli
+from milc.subcommand import config
+
 
 @contextlib.contextmanager
-def parallelize(do_parallel):
+def parallelize():
     """Returns a function that can be used in place of a map() call.
 
     Attempts to use `mpire`, falling back to `multiprocessing` if it's not
     available. If parallelization is not requested, returns the original map()
     function.
     """
-    if not do_parallel:
+
+    # Work out if we've already got a config value for parallel searching
+    if not cli.config.user.parallel_search:
+        parallel_search = True
+    else:
+        parallel_search = cli.config.user.parallel_search
+
+    # If we haven't already written a value, write it to the file
+    if cli.config.user.parallel_search != parallel_search:
+        cli.args.read_only = False
+        cli.config.user.parallel_search = parallel_search
+        config.set_config('user', 'parallel_search', parallel_search)
+        cli.save_config()
+
+    # Non-parallel searches use use `map()`
+    if not parallel_search:
         yield map
         return
 
+    # Prefer mpire's `WorkerPool` if it's available
     with contextlib.suppress(ImportError):
         from mpire import WorkerPool
         with WorkerPool() as pool:
             yield functools.partial(pool.imap_unordered, progress_bar=True)
         return
 
+    # Otherwise fall back to multiprocessing's `Pool`
     with multiprocessing.Pool() as pool:
         yield pool.imap_unordered
+
+
+def parallel_map(*args, **kwargs):
+    """Effectively runs `map()` but executes it in parallel if necessary.
+    """
+    with parallelize() as map_fn:
+        return map_fn(*args, **kwargs)

From d300b555584b767b728d78de2557806443b911d0 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 16:13:28 +1000
Subject: [PATCH 43/84] Actually allow for `False`

---
 lib/python/qmk/util.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py
index 87915d2151df..0fcab3d413fb 100644
--- a/lib/python/qmk/util.py
+++ b/lib/python/qmk/util.py
@@ -18,7 +18,7 @@ def parallelize():
     """
 
     # Work out if we've already got a config value for parallel searching
-    if not cli.config.user.parallel_search:
+    if cli.config.user.parallel_search is None:
         parallel_search = True
     else:
         parallel_search = cli.config.user.parallel_search
@@ -26,9 +26,9 @@ def parallelize():
     # If we haven't already written a value, write it to the file
     if cli.config.user.parallel_search != parallel_search:
         cli.args.read_only = False
-        cli.config.user.parallel_search = parallel_search
         config.set_config('user', 'parallel_search', parallel_search)
         cli.save_config()
+        cli.config.user.parallel_search = parallel_search
 
     # Non-parallel searches use use `map()`
     if not parallel_search:

From 7c3f69a0f6649601de0467d2d413ea1f1b5f7324 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 16:32:08 +1000
Subject: [PATCH 44/84] missing `qmk.json` should not validate as OK

---
 lib/python/qmk/cli/doctor/main.py |  6 ++++--
 lib/python/qmk/userspace.py       | 11 +++++++----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 0c1e4bc0b3b5..69511f12b230 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -13,7 +13,7 @@
 from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
 from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation
 from qmk.commands import in_virtualenv
-from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationException
+from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError
 
 
 def os_tests():
@@ -101,7 +101,9 @@ def userspace_tests(qmk_firmware):
         try:
             qmk_userspace_validate(path)
             cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`')
-        except UserspaceValidationException as err:
+        except FileNotFoundError as err:
+            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Missing `qmk.json`')
+        except UserspaceValidationError as err:
             cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`')
             cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {err}')
 
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index e7f4add1d03c..3753e9b5a374 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -28,6 +28,7 @@ def qmk_userspace_paths():
 def qmk_userspace_validate(path):
     if (path / 'qmk.json').is_file():
         UserspaceDefs(path / 'qmk.json')
+    raise FileNotFoundError('No qmk.json file found.')
 
 
 def detect_qmk_userspace():
@@ -36,8 +37,10 @@ def detect_qmk_userspace():
         try:
             qmk_userspace_validate(test_dir)
             return test_dir
-        except UserspaceValidationException:
-            pass
+        except FileNotFoundError:
+            continue
+        except UserspaceValidationError:
+            continue
     return None
 
 
@@ -47,7 +50,7 @@ def __init__(self, userspace_json: Path):
         self.build_targets = []
         json = json_load(userspace_json)
 
-        exception = UserspaceValidationException()
+        exception = UserspaceValidationError()
         success = False
 
         try:
@@ -102,7 +105,7 @@ def __load_v1(self, json):
                 self.add_target(e[0], e[1])
 
 
-class UserspaceValidationException(Exception):
+class UserspaceValidationError(Exception):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.__exceptions = []

From fdab617403108e5ecf1965691e1b3f084e8afa1f Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 29 Sep 2023 16:33:37 +1000
Subject: [PATCH 45/84] Sigh.

---
 lib/python/qmk/userspace.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index 3753e9b5a374..71fef031ada5 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -28,6 +28,7 @@ def qmk_userspace_paths():
 def qmk_userspace_validate(path):
     if (path / 'qmk.json').is_file():
         UserspaceDefs(path / 'qmk.json')
+        return
     raise FileNotFoundError('No qmk.json file found.')
 
 

From 6566ff904bf3c04bc76c13c64817bda16760d7fb Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Thu, 5 Oct 2023 16:10:10 +1100
Subject: [PATCH 46/84] Additionally search aliases for keymaps when we have
 userspace.

---
 lib/python/qmk/commands.py |  9 ++++++++
 lib/python/qmk/keyboard.py | 16 +++++++++++++
 lib/python/qmk/keymap.py   | 47 +++++++++++++++++++++++---------------
 3 files changed, 54 insertions(+), 18 deletions(-)

diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 5ce9dcf31b98..2d8a29a67b1c 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -99,6 +99,10 @@ def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1
     if dry_run:
         make_command.append('-n')
 
+    from qmk.keymap import locate_keymap
+    keymap_location = locate_keymap(keyboard, keymap)
+    keymap_path = keymap_location.parent
+
     make_command.extend([
         *get_make_parallel_args(parallel),
         '-r',
@@ -115,6 +119,11 @@ def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1
         f'KEYMAP={keymap}',
         'SILENT=false',
         'QMK_BIN="qmk"',
+        f'MAIN_KEYMAP_PATH_1={keymap_path}',
+        f'MAIN_KEYMAP_PATH_2={keymap_path}',
+        f'MAIN_KEYMAP_PATH_3={keymap_path}',
+        f'MAIN_KEYMAP_PATH_4={keymap_path}',
+        f'MAIN_KEYMAP_PATH_5={keymap_path}',
     ])
 
     if cli.config.general.verbose:
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 8cb609aba63b..d5d7308e646f 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -109,6 +109,22 @@ def keyboard_folder(keyboard):
     return keyboard
 
 
+def keyboard_aliases(keyboard):
+    """Returns the list of aliases for the supplied keyboard.
+
+    Includes the keyboard itself.
+    """
+    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
+
+    if keyboard in aliases:
+        keyboard = aliases[keyboard].get('target', keyboard)
+
+    keyboards = set(filter(lambda k: aliases[k].get('target', '') == keyboard, aliases.keys()))
+    keyboards.add(keyboard)
+    keyboards = list(sorted(keyboards))
+    return keyboards
+
+
 def keyboard_folder_or_all(keyboard):
     """Returns the actual keyboard folder.
 
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index 8f52ea30a61a..908c21857038 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -13,7 +13,7 @@
 
 import qmk.path
 from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
-from qmk.keyboard import find_keyboard_from_dir, keyboard_folder
+from qmk.keyboard import find_keyboard_from_dir, keyboard_folder, keyboard_aliases
 from qmk.errors import CppError
 from qmk.info import info_json
 
@@ -430,23 +430,34 @@ def locate_keymap(keyboard, keymap):
     # Check the keyboard folder first, last match wins
     keymap_path = ''
 
-    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]:
-        checked_dirs = ''
-        for dir in keyboard_folder(keyboard).split('/'):
-            if checked_dirs:
-                checked_dirs = '/'.join((checked_dirs, dir))
-            else:
-                checked_dirs = dir
-
-            keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps'
-
-            if (keymap_dir / keymap / 'keymap.c').exists():
-                keymap_path = keymap_dir / keymap / 'keymap.c'
-            if (keymap_dir / keymap / 'keymap.json').exists():
-                keymap_path = keymap_dir / keymap / 'keymap.json'
-
-    if keymap_path:
-        return keymap_path
+    search_dirs = [QMK_FIRMWARE]
+    keyboard_dirs = [keyboard_folder(keyboard)]
+    if HAS_QMK_USERSPACE:
+        # When we've got userspace, check there _last_ as we want them to override anything in the main repo.
+        search_dirs.append(QMK_USERSPACE)
+        # We also want to search for any aliases as QMK's folder structure may have changed, with an alias, but the user
+        # hasn't updated their keymap location yet.
+        keyboard_dirs.extend(keyboard_aliases(keyboard))
+        keyboard_dirs = list(set(keyboard_dirs))
+
+    for search_dir in search_dirs:
+        for keyboard_dir in keyboard_dirs:
+            checked_dirs = ''
+            for dir in keyboard_dir.split('/'):
+                if checked_dirs:
+                    checked_dirs = '/'.join((checked_dirs, dir))
+                else:
+                    checked_dirs = dir
+
+                keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps'
+
+                if (keymap_dir / keymap / 'keymap.c').exists():
+                    keymap_path = keymap_dir / keymap / 'keymap.c'
+                if (keymap_dir / keymap / 'keymap.json').exists():
+                    keymap_path = keymap_dir / keymap / 'keymap.json'
+
+        if keymap_path:
+            return keymap_path
 
     # Check community layouts as a fallback
     info = info_json(keyboard)

From dbce00718140c26b3277c82e0fae84a7f6a4d610 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Thu, 5 Oct 2023 16:33:19 +1100
Subject: [PATCH 47/84] Fixup `qmk new-keymap`

---
 lib/python/qmk/cli/new/keymap.py |  3 ---
 lib/python/qmk/path.py           | 19 +++++++++++++++++--
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index f7ffcab0e992..c2c1c975329b 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -56,9 +56,6 @@ def new_keymap(cli):
     keymap_path_default = keymap(kb_name, 'default')
     keymap_path_new = keymaps_dirs[0] / user_name
 
-    if HAS_QMK_USERSPACE:
-        keymap_path_new = Path(QMK_USERSPACE) / keymaps_dirs[0].relative_to(QMK_FIRMWARE) / user_name
-
     if not keymap_path_default.exists():
         cli.log.error(f'Default keymap {{fg_cyan}}{keymap_path_default}{{fg_reset}} does not exist!')
         return False
diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py
index 7a6655ff4727..4f8beee9a6e7 100644
--- a/lib/python/qmk/path.py
+++ b/lib/python/qmk/path.py
@@ -60,13 +60,28 @@ def keymaps(keyboard_name):
     keyboard_folder = keyboard(keyboard_name)
     found_dirs = []
 
-    for root_dir in [QMK_USERSPACE, QMK_FIRMWARE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]:
-        this_keyboard_folder = root_dir / keyboard_folder
+    if HAS_QMK_USERSPACE:
+        this_keyboard_folder = Path(QMK_USERSPACE) / keyboard_folder
         for _ in range(MAX_KEYBOARD_SUBFOLDERS):
             if (this_keyboard_folder / 'keymaps').exists():
                 found_dirs.append((this_keyboard_folder / 'keymaps').resolve())
 
             this_keyboard_folder = this_keyboard_folder.parent
+            if this_keyboard_folder.resolve() == QMK_USERSPACE.resolve():
+                break
+
+        # We don't have any relevant keymap directories in userspace, so we'll use the fully-qualified path instead.
+        if len(found_dirs) == 0:
+            found_dirs.append((QMK_USERSPACE / keyboard_folder / 'keymaps').resolve())
+
+    this_keyboard_folder = QMK_FIRMWARE / keyboard_folder
+    for _ in range(MAX_KEYBOARD_SUBFOLDERS):
+        if (this_keyboard_folder / 'keymaps').exists():
+            found_dirs.append((this_keyboard_folder / 'keymaps').resolve())
+
+        this_keyboard_folder = this_keyboard_folder.parent
+        if this_keyboard_folder.resolve() == QMK_FIRMWARE.resolve():
+            break
 
     if len(found_dirs) > 0:
         return found_dirs

From b8fd11e16149f07ab8c7040a104624cf67086ee6 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 7 Oct 2023 20:43:40 +1100
Subject: [PATCH 48/84] Update lib/python/qmk/util.py

Co-authored-by: Duncan Sutherland <dunk2k_2000@hotmail.com>
---
 lib/python/qmk/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py
index 0fcab3d413fb..b1c4c7366b1a 100644
--- a/lib/python/qmk/util.py
+++ b/lib/python/qmk/util.py
@@ -30,7 +30,7 @@ def parallelize():
         cli.save_config()
         cli.config.user.parallel_search = parallel_search
 
-    # Non-parallel searches use use `map()`
+    # Non-parallel searches use `map()`
     if not parallel_search:
         yield map
         return

From 0888ee379981cb3b8475ea281b59ec4401015592 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sat, 7 Oct 2023 22:05:09 +1100
Subject: [PATCH 49/84] Start refactoring build targets.

---
 lib/python/qmk/build_targets.py               | 123 ++++++++++++++++++
 lib/python/qmk/cli/compile.py                 |  44 ++-----
 .../qmk/cli/generate/compilation_database.py  |  14 +-
 3 files changed, 140 insertions(+), 41 deletions(-)
 create mode 100644 lib/python/qmk/build_targets.py

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
new file mode 100644
index 000000000000..628db19ea1e2
--- /dev/null
+++ b/lib/python/qmk/build_targets.py
@@ -0,0 +1,123 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from typing import List
+from pathlib import Path
+from milc import cli
+from qmk.constants import QMK_FIRMWARE
+from qmk.commands import _find_make, get_make_parallel_args, parse_configurator_json, create_make_target
+from qmk.cli.generate.compilation_database import write_compilation_database
+
+
+class BuildTarget:
+    def __init__(self, keyboard: str, keymap: str):
+        self._keyboard = keyboard
+        self._keymap = keymap
+        self._parallel = 1
+        self._clean = False
+        self._compiledb = False
+
+    def __str__(self):
+        return f'{self.keyboard}:{self.keymap}'
+
+    def __repr__(self):
+        return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'
+
+    def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None:
+        if parallel is not None:
+            self._parallel = parallel
+        if clean is not None:
+            self._clean = clean
+        if compiledb is not None:
+            self._compiledb = compiledb
+
+    @property
+    def keyboard(self) -> str:
+        return self._keyboard
+
+    @property
+    def keymap(self) -> str:
+        return self._keymap
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        raise NotImplementedError("prepare_build() not implemented in base class")
+
+    def compile_commands(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        raise NotImplementedError("compile_commands() not implemented in base class")
+
+    def generate_compilation_database(self, skip_clean: bool = False, **env_vars) -> None:
+        self.prepare_build(**env_vars)
+        command = self.compile_commands(dry_run=True, **env_vars)
+        write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars)
+
+    def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        if self._clean or self._compiledb:
+            cli.run(create_make_target("clean", dry_run=dry_run, parallel=1, **env_vars), capture_output=False)
+
+        self.prepare_build(build_target=build_target, **env_vars)
+
+        if self._compiledb and not dry_run:
+            self.generate_compilation_database(skip_clean=True, **env_vars)
+
+        command = self.compile_commands(build_target=build_target, **env_vars)
+        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
+        if not dry_run:
+            cli.echo('\n')
+            ret = cli.run(command, capture_output=False)
+            if ret.returncode:
+                return ret.returncode
+
+
+class KeyboardKeymapBuildTarget(BuildTarget):
+    def __init__(self, keyboard: str, keymap: str):
+        super().__init__(keyboard, keymap)
+
+    def __repr__(self):
+        return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        pass
+
+    def compile_commands(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        """This needs to be rewritten to do a full `make` invocation, rather than the old syntax!
+
+        See commands.py for actual implementation.
+        """
+
+        compile_args = [
+            _find_make(),
+            *get_make_parallel_args(self._parallel),
+        ]
+
+        if dry_run:
+            compile_args.append('-n')
+
+        if not build_target:
+            compile_args.append(f'{self.keyboard}:{self.keymap}')
+        else:
+            compile_args.append(f'{self.keyboard}:{self.keymap}:{build_target}')
+
+        for key, value in env_vars.items():
+            compile_args.append(f'{key}={value}')
+
+        if cli.config.general.verbose:
+            compile_args.append('VERBOSE=true')
+
+        return compile_args
+
+
+class JsonKeymapBuildTarget(BuildTarget):
+    def __init__(self, json_path: Path, json: dict = None):
+        self.json_path = Path(json_path)
+        if json is None:
+            json = parse_configurator_json(self.json_path)
+        self.json = json
+        super().__init__(self.json['keyboard'], self.json['keymap'])
+
+    def __repr__(self):
+        return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        pass
+
+    def compile_commands(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        return []
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 2478ea1fed74..05a6b965da33 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -9,16 +9,13 @@
 import qmk.path
 from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
+from qmk.commands import build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
 from qmk.keymap import keymap_completer, locate_keymap
-from qmk.cli.generate.compilation_database import write_compilation_database
+from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
 
 
 def _is_keymap_target(keyboard, keymap):
-    if keymap == 'all':
-        return True
-
     if locate_keymap(keyboard, keymap):
         return True
 
@@ -53,47 +50,24 @@ def compile(cli):
     # Build the environment vars
     envs = build_environment(cli.args.env)
 
-    # Determine the compile command
-    commands = []
-
-    current_keyboard = None
-    current_keymap = None
+    # Handler for the build target
+    target = None
 
     if cli.args.filename:
-        # If a configurator JSON was provided generate a keymap and compile it
-        user_keymap = parse_configurator_json(cli.args.filename)
-        commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
+        target = JsonKeymapBuildTarget(cli.args.filename)
 
     elif cli.config.compile.keyboard and cli.config.compile.keymap:
-        # Generate the make command for a specific keyboard/keymap.
         if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
             cli.log.error('Invalid keymap argument.')
             cli.print_help()
             return False
 
-        if cli.args.clean:
-            commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
-        commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
+        target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap)
 
-        current_keyboard = cli.config.compile.keyboard
-        current_keymap = cli.config.compile.keymap
-
-    if not commands:
+    if not target:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
         cli.print_help()
         return False
 
-    if cli.args.compiledb:
-        if current_keyboard is None or current_keymap is None:
-            cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.')
-            cli.print_help()
-            return False
-        write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json')
-
-    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
-    if not cli.args.dry_run:
-        cli.echo('\n')
-        for command in commands:
-            ret = cli.run(command, capture_output=False)
-            if ret.returncode:
-                return ret.returncode
+    target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
+    target.compile(**envs)
diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py
index 9e5c266516d0..3b9e150708b9 100755
--- a/lib/python/qmk/cli/generate/compilation_database.py
+++ b/lib/python/qmk/cli/generate/compilation_database.py
@@ -12,7 +12,7 @@
 
 from milc import cli, MILC
 
-from qmk.commands import create_make_command
+from qmk.commands import create_make_command, create_make_target
 from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
@@ -76,9 +76,10 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
     return records
 
 
-def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> bool:
+def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None) -> bool:
     # Generate the make command for a specific keyboard/keymap.
-    command = create_make_command(keyboard, keymap, dry_run=True)
+    if not command:
+        command = create_make_command(keyboard, keymap, dry_run=True)
 
     if not command:
         cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
@@ -90,9 +91,10 @@ def write_compilation_database(keyboard: str, keymap: str, output_path: Path) ->
     env.pop("MAKEFLAGS", None)
 
     # re-use same executable as the main make invocation (might be gmake)
-    clean_command = [command[0], 'clean']
-    cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
-    cli.run(clean_command, capture_output=False, check=True, env=env)
+    if not skip_clean:
+        clean_command = create_make_target("clean")
+        cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
+        cli.run(clean_command, capture_output=False, check=True, env=env)
 
     cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command))
 

From 88bdabb985ecc57fbd1cdc448566fafd6fe93483 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 8 Oct 2023 18:38:43 +1100
Subject: [PATCH 50/84] More refactoring.

---
 lib/python/qmk/build_targets.py | 152 +++++++++++++++++++++++++-------
 lib/python/qmk/cli/compile.py   |  16 ++--
 2 files changed, 128 insertions(+), 40 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 628db19ea1e2..d082406cfbdc 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -1,16 +1,19 @@
 # Copyright 2023 Nick Brassel (@tzarc)
 # SPDX-License-Identifier: GPL-2.0-or-later
+import json
 from typing import List
 from pathlib import Path
 from milc import cli
-from qmk.constants import QMK_FIRMWARE
+from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
 from qmk.commands import _find_make, get_make_parallel_args, parse_configurator_json, create_make_target
+from qmk.keymap import locate_keymap
 from qmk.cli.generate.compilation_database import write_compilation_database
 
 
 class BuildTarget:
     def __init__(self, keyboard: str, keymap: str):
         self._keyboard = keyboard
+        self._keyboard_safe = keyboard.replace('/', '_')
         self._keymap = keymap
         self._parallel = 1
         self._clean = False
@@ -41,24 +44,23 @@ def keymap(self) -> str:
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         raise NotImplementedError("prepare_build() not implemented in base class")
 
-    def compile_commands(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
-        raise NotImplementedError("compile_commands() not implemented in base class")
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        raise NotImplementedError("compile_command() not implemented in base class")
 
-    def generate_compilation_database(self, skip_clean: bool = False, **env_vars) -> None:
-        self.prepare_build(**env_vars)
-        command = self.compile_commands(dry_run=True, **env_vars)
+    def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None:
+        self.prepare_build(build_target=build_target, **env_vars)
+        command = self.compile_command(build_target=build_target, dry_run=True, **env_vars)
         write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars)
 
     def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         if self._clean or self._compiledb:
             cli.run(create_make_target("clean", dry_run=dry_run, parallel=1, **env_vars), capture_output=False)
 
-        self.prepare_build(build_target=build_target, **env_vars)
-
         if self._compiledb and not dry_run:
-            self.generate_compilation_database(skip_clean=True, **env_vars)
+            self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars)
 
-        command = self.compile_commands(build_target=build_target, **env_vars)
+        self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars)
+        command = self.compile_command(build_target=build_target, **env_vars)
         cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
         if not dry_run:
             cli.echo('\n')
@@ -77,47 +79,137 @@ def __repr__(self):
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         pass
 
-    def compile_commands(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
-        """This needs to be rewritten to do a full `make` invocation, rather than the old syntax!
-
-        See commands.py for actual implementation.
-        """
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        verbose = 'true' if cli.config.general.verbose else 'false'
+        color = 'true' if cli.config.general.color else 'false'
 
         compile_args = [
             _find_make(),
             *get_make_parallel_args(self._parallel),
+            '-r',
+            '-R',
+            '-f',
+            'builddefs/build_keyboard.mk',
         ]
 
+        if not cli.config.general.verbose:
+            compile_args.append('-s')
+
         if dry_run:
             compile_args.append('-n')
 
-        if not build_target:
-            compile_args.append(f'{self.keyboard}:{self.keymap}')
-        else:
-            compile_args.append(f'{self.keyboard}:{self.keymap}:{build_target}')
+        if build_target:
+            compile_args.append(build_target)
+
+        target = f'{self._keyboard_safe}_{self.keymap}'
+        intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.keymap}')
+        keymap_file = Path(locate_keymap(self.keyboard, self.keymap))
+        keymap_dir = keymap_file.parent
+
+        compile_args.extend([
+            f'KEYBOARD={self.keyboard}',
+            f'KEYMAP={self.keymap}',
+            f'KEYBOARD_FILESAFE={self._keyboard_safe}',
+            f'TARGET={target}',
+            f'INTERMEDIATE_OUTPUT={intermediate_output}',
+            f'KEYMAP_C={keymap_file}',
+            f'KEYMAP_PATH={keymap_dir}',
+            f'VERBOSE={verbose}',
+            f'COLOR={color}',
+            'SILENT=false',
+            'QMK_BIN="qmk"',
+        ])
 
         for key, value in env_vars.items():
             compile_args.append(f'{key}={value}')
 
-        if cli.config.general.verbose:
-            compile_args.append('VERBOSE=true')
-
         return compile_args
 
 
 class JsonKeymapBuildTarget(BuildTarget):
-    def __init__(self, json_path: Path, json: dict = None):
-        self.json_path = Path(json_path)
-        if json is None:
-            json = parse_configurator_json(self.json_path)
-        self.json = json
+    def __init__(self, json_path):
+        if isinstance(json_path, Path):
+            self.json_path = json_path
+        else:
+            self.json_path = None
+
+        self.json = parse_configurator_json(json_path)  # Will load from stdin if provided
+
+        # In case the user passes a keymap.json from a keymap directory directly to the CLI.
+        # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
+        self.json["keymap"] = self.json.get("keymap", "default_json")
+
         super().__init__(self.json['keyboard'], self.json['keymap'])
 
     def __repr__(self):
         return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
 
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
-        pass
+        intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.json["keymap"]}')
+        keymap_dir = intermediate_output / 'src'
+        keymap_json = keymap_dir / 'keymap.json'
+
+        # begin with making the deepest folder in the tree
+        keymap_dir.mkdir(exist_ok=True, parents=True)
+
+        # Compare minified to ensure consistent comparison
+        new_content = json.dumps(self.json, separators=(',', ':'))
+        if keymap_json.exists():
+            old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
+            if old_content == new_content:
+                new_content = None
+
+        # Write the keymap.json file if different
+        if new_content:
+            keymap_json.write_text(new_content, encoding='utf-8')
 
-    def compile_commands(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
-        return []
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        verbose = 'true' if cli.config.general.verbose else 'false'
+        color = 'true' if cli.config.general.color else 'false'
+
+        compile_args = [
+            _find_make(),
+            *get_make_parallel_args(self._parallel),
+            '-r',
+            '-R',
+            '-f',
+            'builddefs/build_keyboard.mk',
+        ]
+
+        if not cli.config.general.verbose:
+            compile_args.append('-s')
+
+        if dry_run:
+            compile_args.append('-n')
+
+        if build_target:
+            compile_args.append(build_target)
+
+        target = f'{self._keyboard_safe}_{self.keymap}'
+        intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.json["keymap"]}')
+        keymap_dir = intermediate_output / 'src'
+        keymap_json = keymap_dir / 'keymap.json'
+
+        compile_args.extend([
+            f'KEYBOARD={self.keyboard}',
+            f'KEYMAP={self.keymap}',
+            f'KEYBOARD_FILESAFE={self._keyboard_safe}',
+            f'TARGET={target}',
+            f'INTERMEDIATE_OUTPUT={intermediate_output}',
+            f'MAIN_KEYMAP_PATH_1={intermediate_output}',
+            f'MAIN_KEYMAP_PATH_2={intermediate_output}',
+            f'MAIN_KEYMAP_PATH_3={intermediate_output}',
+            f'MAIN_KEYMAP_PATH_4={intermediate_output}',
+            f'MAIN_KEYMAP_PATH_5={intermediate_output}',
+            f'KEYMAP_JSON={keymap_json}',
+            f'KEYMAP_PATH={keymap_dir}',
+            f'VERBOSE={verbose}',
+            f'COLOR={color}',
+            'SILENT=false',
+            'QMK_BIN="qmk"',
+        ])
+
+        for key, value in env_vars.items():
+            compile_args.append(f'{key}={value}')
+
+        return compile_args
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 05a6b965da33..8de2d5ed1e5d 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -7,7 +7,6 @@
 from milc import cli
 
 import qmk.path
-from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
@@ -15,13 +14,6 @@
 from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
 
 
-def _is_keymap_target(keyboard, keymap):
-    if locate_keymap(keyboard, keymap):
-        return True
-
-    return False
-
-
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
 @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
@@ -29,6 +21,7 @@ def _is_keymap_target(keyboard, keymap):
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
+@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")
 @cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.")
 @cli.subcommand('Compile a QMK Firmware.')
 @automagic_keyboard
@@ -54,14 +47,17 @@ def compile(cli):
     target = None
 
     if cli.args.filename:
+        # if we were given a filename, assume we have a json build target
         target = JsonKeymapBuildTarget(cli.args.filename)
 
     elif cli.config.compile.keyboard and cli.config.compile.keymap:
-        if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
+        # if we got a keyboard and keymap, attempt to find it
+        if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap):
             cli.log.error('Invalid keymap argument.')
             cli.print_help()
             return False
 
+        # If we got here, then we have a valid keyboard and keymap for a build target
         target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap)
 
     if not target:
@@ -70,4 +66,4 @@ def compile(cli):
         return False
 
     target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
-    target.compile(**envs)
+    target.compile(cli.args.target, **envs)

From 4fa09b53b60e7b647edbb05b85f54dc8a36775ce Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 8 Oct 2023 18:52:58 +1100
Subject: [PATCH 51/84] Fix tests.

---
 lib/python/qmk/build_targets.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index d082406cfbdc..b2a4c540ab09 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -112,8 +112,6 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
             f'KEYBOARD_FILESAFE={self._keyboard_safe}',
             f'TARGET={target}',
             f'INTERMEDIATE_OUTPUT={intermediate_output}',
-            f'KEYMAP_C={keymap_file}',
-            f'KEYMAP_PATH={keymap_dir}',
             f'VERBOSE={verbose}',
             f'COLOR={color}',
             'SILENT=false',

From 8501251b27636b4074dc8ae3e30f8c585a27568c Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 8 Oct 2023 20:42:12 +1100
Subject: [PATCH 52/84] Fix tests.

---
 lib/python/qmk/build_targets.py | 1 -
 lib/python/qmk/cli/compile.py   | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index b2a4c540ab09..a008e0e8c4b1 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -104,7 +104,6 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
         target = f'{self._keyboard_safe}_{self.keymap}'
         intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.keymap}')
         keymap_file = Path(locate_keymap(self.keyboard, self.keymap))
-        keymap_dir = keymap_file.parent
 
         compile_args.extend([
             f'KEYBOARD={self.keyboard}',
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 8de2d5ed1e5d..71c1dec1621f 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -66,4 +66,4 @@ def compile(cli):
         return False
 
     target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
-    target.compile(cli.args.target, **envs)
+    target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs)

From 4b3a454fc03a0530e8c37bba21d9b46b2bd73324 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 9 Oct 2023 20:21:05 +1100
Subject: [PATCH 53/84] Refactoring.

---
 lib/python/qmk/build_targets.py | 99 +++++++++++++++------------------
 1 file changed, 44 insertions(+), 55 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index a008e0e8c4b1..906d35a23655 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -6,7 +6,6 @@
 from milc import cli
 from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
 from qmk.commands import _find_make, get_make_parallel_args, parse_configurator_json, create_make_target
-from qmk.keymap import locate_keymap
 from qmk.cli.generate.compilation_database import write_compilation_database
 
 
@@ -41,6 +40,42 @@ def keyboard(self) -> str:
     def keymap(self) -> str:
         return self._keymap
 
+    def _common_make_args(self):
+        compile_args = [
+            _find_make(),
+            *get_make_parallel_args(self._parallel),
+            '-r',
+            '-R',
+            '-f',
+            'builddefs/build_keyboard.mk',
+        ]
+
+        if not cli.config.general.verbose:
+            compile_args.append('-s')
+
+        return compile_args
+
+    def _common_make_vars(self):
+        verbose = 'true' if cli.config.general.verbose else 'false'
+        color = 'true' if cli.config.general.color else 'false'
+
+        target = f'{self._keyboard_safe}_{self.keymap}'
+        intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.keymap}')
+
+        make_args = [
+            f'KEYBOARD={self.keyboard}',
+            f'KEYMAP={self.keymap}',
+            f'KEYBOARD_FILESAFE={self._keyboard_safe}',
+            f'TARGET={target}',
+            f'INTERMEDIATE_OUTPUT={intermediate_output}',
+            f'VERBOSE={verbose}',
+            f'COLOR={color}',
+            'SILENT=false',
+            'QMK_BIN="qmk"',
+        ]
+
+        return make_args
+
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         raise NotImplementedError("prepare_build() not implemented in base class")
 
@@ -54,7 +89,9 @@ def generate_compilation_database(self, build_target: str = None, skip_clean: bo
 
     def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         if self._clean or self._compiledb:
-            cli.run(create_make_target("clean", dry_run=dry_run, parallel=1, **env_vars), capture_output=False)
+            command = create_make_target("clean", dry_run=dry_run, parallel=1, **env_vars)
+            cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command))
+            cli.run(command, capture_output=False)
 
         if self._compiledb and not dry_run:
             self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars)
@@ -80,20 +117,7 @@ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_v
         pass
 
     def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
-        verbose = 'true' if cli.config.general.verbose else 'false'
-        color = 'true' if cli.config.general.color else 'false'
-
-        compile_args = [
-            _find_make(),
-            *get_make_parallel_args(self._parallel),
-            '-r',
-            '-R',
-            '-f',
-            'builddefs/build_keyboard.mk',
-        ]
-
-        if not cli.config.general.verbose:
-            compile_args.append('-s')
+        compile_args = self._common_make_args()
 
         if dry_run:
             compile_args.append('-n')
@@ -101,21 +125,7 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
         if build_target:
             compile_args.append(build_target)
 
-        target = f'{self._keyboard_safe}_{self.keymap}'
-        intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.keymap}')
-        keymap_file = Path(locate_keymap(self.keyboard, self.keymap))
-
-        compile_args.extend([
-            f'KEYBOARD={self.keyboard}',
-            f'KEYMAP={self.keymap}',
-            f'KEYBOARD_FILESAFE={self._keyboard_safe}',
-            f'TARGET={target}',
-            f'INTERMEDIATE_OUTPUT={intermediate_output}',
-            f'VERBOSE={verbose}',
-            f'COLOR={color}',
-            'SILENT=false',
-            'QMK_BIN="qmk"',
-        ])
+        compile_args.extend(self._common_make_vars())
 
         for key, value in env_vars.items():
             compile_args.append(f'{key}={value}')
@@ -161,20 +171,7 @@ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_v
             keymap_json.write_text(new_content, encoding='utf-8')
 
     def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
-        verbose = 'true' if cli.config.general.verbose else 'false'
-        color = 'true' if cli.config.general.color else 'false'
-
-        compile_args = [
-            _find_make(),
-            *get_make_parallel_args(self._parallel),
-            '-r',
-            '-R',
-            '-f',
-            'builddefs/build_keyboard.mk',
-        ]
-
-        if not cli.config.general.verbose:
-            compile_args.append('-s')
+        compile_args = self._common_make_args()
 
         if dry_run:
             compile_args.append('-n')
@@ -182,17 +179,13 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
         if build_target:
             compile_args.append(build_target)
 
-        target = f'{self._keyboard_safe}_{self.keymap}'
+        compile_args.extend(self._common_make_vars())
+
         intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.json["keymap"]}')
         keymap_dir = intermediate_output / 'src'
         keymap_json = keymap_dir / 'keymap.json'
 
         compile_args.extend([
-            f'KEYBOARD={self.keyboard}',
-            f'KEYMAP={self.keymap}',
-            f'KEYBOARD_FILESAFE={self._keyboard_safe}',
-            f'TARGET={target}',
-            f'INTERMEDIATE_OUTPUT={intermediate_output}',
             f'MAIN_KEYMAP_PATH_1={intermediate_output}',
             f'MAIN_KEYMAP_PATH_2={intermediate_output}',
             f'MAIN_KEYMAP_PATH_3={intermediate_output}',
@@ -200,10 +193,6 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
             f'MAIN_KEYMAP_PATH_5={intermediate_output}',
             f'KEYMAP_JSON={keymap_json}',
             f'KEYMAP_PATH={keymap_dir}',
-            f'VERBOSE={verbose}',
-            f'COLOR={color}',
-            'SILENT=false',
-            'QMK_BIN="qmk"',
         ])
 
         for key, value in env_vars.items():

From b4fab9f79a49f36cc427bdefa11ce9cf97ebbd83 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 9 Oct 2023 20:29:01 +1100
Subject: [PATCH 54/84] Refactoring.

---
 lib/python/qmk/build_targets.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 906d35a23655..dfc34f223682 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -17,6 +17,8 @@ def __init__(self, keyboard: str, keymap: str):
         self._parallel = 1
         self._clean = False
         self._compiledb = False
+        self._target = f'{self._keyboard_safe}_{self.keymap}'
+        self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
 
     def __str__(self):
         return f'{self.keyboard}:{self.keymap}'
@@ -59,15 +61,12 @@ def _common_make_vars(self):
         verbose = 'true' if cli.config.general.verbose else 'false'
         color = 'true' if cli.config.general.color else 'false'
 
-        target = f'{self._keyboard_safe}_{self.keymap}'
-        intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.keymap}')
-
         make_args = [
             f'KEYBOARD={self.keyboard}',
             f'KEYMAP={self.keymap}',
             f'KEYBOARD_FILESAFE={self._keyboard_safe}',
-            f'TARGET={target}',
-            f'INTERMEDIATE_OUTPUT={intermediate_output}',
+            f'TARGET={self._target}',
+            f'INTERMEDIATE_OUTPUT={self._intermediate_output}',
             f'VERBOSE={verbose}',
             f'COLOR={color}',
             'SILENT=false',
@@ -152,8 +151,7 @@ def __repr__(self):
         return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
 
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
-        intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.json["keymap"]}')
-        keymap_dir = intermediate_output / 'src'
+        keymap_dir = self._intermediate_output / 'src'
         keymap_json = keymap_dir / 'keymap.json'
 
         # begin with making the deepest folder in the tree
@@ -181,16 +179,15 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
 
         compile_args.extend(self._common_make_vars())
 
-        intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._keyboard_safe}_{self.json["keymap"]}')
-        keymap_dir = intermediate_output / 'src'
+        keymap_dir = self._intermediate_output / 'src'
         keymap_json = keymap_dir / 'keymap.json'
 
         compile_args.extend([
-            f'MAIN_KEYMAP_PATH_1={intermediate_output}',
-            f'MAIN_KEYMAP_PATH_2={intermediate_output}',
-            f'MAIN_KEYMAP_PATH_3={intermediate_output}',
-            f'MAIN_KEYMAP_PATH_4={intermediate_output}',
-            f'MAIN_KEYMAP_PATH_5={intermediate_output}',
+            f'MAIN_KEYMAP_PATH_1={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_2={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_3={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_4={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_5={self._intermediate_output}',
             f'KEYMAP_JSON={keymap_json}',
             f'KEYMAP_PATH={keymap_dir}',
         ])

From 5b62587a4bfa764bef781eebff8dceeb6767233d Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 9 Oct 2023 20:34:50 +1100
Subject: [PATCH 55/84] Refactoring.

---
 lib/python/qmk/build_targets.py | 38 +++++++++++----------------------
 1 file changed, 12 insertions(+), 26 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index dfc34f223682..3c219d0b3284 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -42,7 +42,7 @@ def keyboard(self) -> str:
     def keymap(self) -> str:
         return self._keymap
 
-    def _common_make_args(self):
+    def _common_make_args(self, dry_run: bool = False, build_target: str = None):
         compile_args = [
             _find_make(),
             *get_make_parallel_args(self._parallel),
@@ -55,13 +55,16 @@ def _common_make_args(self):
         if not cli.config.general.verbose:
             compile_args.append('-s')
 
-        return compile_args
-
-    def _common_make_vars(self):
         verbose = 'true' if cli.config.general.verbose else 'false'
         color = 'true' if cli.config.general.color else 'false'
 
-        make_args = [
+        if dry_run:
+            compile_args.append('-n')
+
+        if build_target:
+            compile_args.append(build_target)
+
+        compile_args.extend([
             f'KEYBOARD={self.keyboard}',
             f'KEYMAP={self.keymap}',
             f'KEYBOARD_FILESAFE={self._keyboard_safe}',
@@ -71,9 +74,9 @@ def _common_make_vars(self):
             f'COLOR={color}',
             'SILENT=false',
             'QMK_BIN="qmk"',
-        ]
+        ])
 
-        return make_args
+        return compile_args
 
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         raise NotImplementedError("prepare_build() not implemented in base class")
@@ -116,16 +119,7 @@ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_v
         pass
 
     def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
-        compile_args = self._common_make_args()
-
-        if dry_run:
-            compile_args.append('-n')
-
-        if build_target:
-            compile_args.append(build_target)
-
-        compile_args.extend(self._common_make_vars())
-
+        compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
         for key, value in env_vars.items():
             compile_args.append(f'{key}={value}')
 
@@ -169,15 +163,7 @@ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_v
             keymap_json.write_text(new_content, encoding='utf-8')
 
     def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
-        compile_args = self._common_make_args()
-
-        if dry_run:
-            compile_args.append('-n')
-
-        if build_target:
-            compile_args.append(build_target)
-
-        compile_args.extend(self._common_make_vars())
+        compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
 
         keymap_dir = self._intermediate_output / 'src'
         keymap_json = keymap_dir / 'keymap.json'

From 24cfc17ff80137dbb879d4d779a0b35f24880c17 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 9 Oct 2023 20:42:57 +1100
Subject: [PATCH 56/84] Refactoring.

---
 lib/python/qmk/build_targets.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 3c219d0b3284..14e6a989a3b3 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -1,6 +1,7 @@
 # Copyright 2023 Nick Brassel (@tzarc)
 # SPDX-License-Identifier: GPL-2.0-or-later
 import json
+import shutil
 from typing import List
 from pathlib import Path
 from milc import cli
@@ -19,6 +20,7 @@ def __init__(self, keyboard: str, keymap: str):
         self._compiledb = False
         self._target = f'{self._keyboard_safe}_{self.keymap}'
         self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
+        self._generated_files_path = self._intermediate_output / 'src'
 
     def __str__(self):
         return f'{self.keyboard}:{self.keymap}'
@@ -122,7 +124,6 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
         compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
         for key, value in env_vars.items():
             compile_args.append(f'{key}={value}')
-
         return compile_args
 
 
@@ -141,41 +142,40 @@ def __init__(self, json_path):
 
         super().__init__(self.json['keyboard'], self.json['keymap'])
 
+        self._keymap_json = self._generated_files_path / 'keymap.json'
+
     def __repr__(self):
         return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
 
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
-        keymap_dir = self._intermediate_output / 'src'
-        keymap_json = keymap_dir / 'keymap.json'
+        if self._clean:
+            if self._intermediate_output.exists():
+                shutil.rmtree(self._intermediate_output)
 
         # begin with making the deepest folder in the tree
-        keymap_dir.mkdir(exist_ok=True, parents=True)
+        self._generated_files_path.mkdir(exist_ok=True, parents=True)
 
         # Compare minified to ensure consistent comparison
         new_content = json.dumps(self.json, separators=(',', ':'))
-        if keymap_json.exists():
-            old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
+        if self._keymap_json.exists():
+            old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
             if old_content == new_content:
                 new_content = None
 
         # Write the keymap.json file if different
         if new_content:
-            keymap_json.write_text(new_content, encoding='utf-8')
+            self._keymap_json.write_text(new_content, encoding='utf-8')
 
     def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
         compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
-
-        keymap_dir = self._intermediate_output / 'src'
-        keymap_json = keymap_dir / 'keymap.json'
-
         compile_args.extend([
             f'MAIN_KEYMAP_PATH_1={self._intermediate_output}',
             f'MAIN_KEYMAP_PATH_2={self._intermediate_output}',
             f'MAIN_KEYMAP_PATH_3={self._intermediate_output}',
             f'MAIN_KEYMAP_PATH_4={self._intermediate_output}',
             f'MAIN_KEYMAP_PATH_5={self._intermediate_output}',
-            f'KEYMAP_JSON={keymap_json}',
-            f'KEYMAP_PATH={keymap_dir}',
+            f'KEYMAP_JSON={self._keymap_json}',
+            f'KEYMAP_PATH={self._generated_files_path}',
         ])
 
         for key, value in env_vars.items():

From 5c68b6132046183b9448fc7682f49820d2bee08f Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 9 Oct 2023 21:06:45 +1100
Subject: [PATCH 57/84] Commentary.

---
 lib/python/qmk/build_targets.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 14e6a989a3b3..a05d9a33de99 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -122,8 +122,10 @@ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_v
 
     def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
         compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
+
         for key, value in env_vars.items():
             compile_args.append(f'{key}={value}')
+
         return compile_args
 
 
@@ -162,7 +164,8 @@ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_v
             if old_content == new_content:
                 new_content = None
 
-        # Write the keymap.json file if different
+        # Write the keymap.json file if different so timestamps are only updated
+        # if the content changes -- running `make` won't treat it as modified.
         if new_content:
             self._keymap_json.write_text(new_content, encoding='utf-8')
 

From bc8ff2d907be4bce3af3e389183057f8690b1aa6 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 9 Oct 2023 22:52:07 +1100
Subject: [PATCH 58/84] Refactoring.

---
 lib/python/qmk/build_targets.py               |   4 +-
 lib/python/qmk/cli/flash.py                   |  35 ++---
 .../qmk/cli/generate/compilation_database.py  |   6 +-
 lib/python/qmk/cli/mass_compile.py            |   4 +-
 lib/python/qmk/commands.py                    | 131 +-----------------
 5 files changed, 25 insertions(+), 155 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index a05d9a33de99..7097e25e3133 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -6,7 +6,7 @@
 from pathlib import Path
 from milc import cli
 from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
-from qmk.commands import _find_make, get_make_parallel_args, parse_configurator_json, create_make_target
+from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json, create_make_target
 from qmk.cli.generate.compilation_database import write_compilation_database
 
 
@@ -46,7 +46,7 @@ def keymap(self) -> str:
 
     def _common_make_args(self, dry_run: bool = False, build_target: str = None):
         compile_args = [
-            _find_make(),
+            find_make(),
             *get_make_parallel_args(self._parallel),
             '-r',
             '-R',
diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
index 8724f26889b2..89c01c280c50 100644
--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -4,15 +4,17 @@
 A bootloader must be specified.
 """
 from argcomplete.completers import FilesCompleter
+from pathlib import Path
 
 from milc import cli
 
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
+from qmk.commands import build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
 from qmk.keymap import keymap_completer, locate_keymap
 from qmk.flashers import flasher
+from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
 
 
 def _is_keymap_target(keyboard, keymap):
@@ -89,7 +91,7 @@ def flash(cli):
 
     If bootloader is omitted the make system will use the configured bootloader for that keyboard.
     """
-    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
+    if cli.args.filename and isinstance(cli.args.filename, Path) and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
         return _flash_binary(cli.args.filename, cli.args.mcu)
 
     if cli.args.bootloaders:
@@ -98,34 +100,27 @@ def flash(cli):
     # Build the environment vars
     envs = build_environment(cli.args.env)
 
-    # Determine the compile command
-    commands = []
+    # Handler for the build target
+    target = None
 
     if cli.args.filename:
-        # If a configurator JSON was provided generate a keymap and compile it
-        user_keymap = parse_configurator_json(cli.args.filename)
-        commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)]
+        # if we were given a filename, assume we have a json build target
+        target = JsonKeymapBuildTarget(cli.args.filename)
 
     elif cli.config.flash.keyboard and cli.config.flash.keymap:
-        # Generate the make command for a specific keyboard/keymap.
-        if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
+        # if we got a keyboard and keymap, attempt to find it
+        if not locate_keymap(cli.config.flash.keyboard, cli.config.flash.keymap):
             cli.log.error('Invalid keymap argument.')
             cli.print_help()
             return False
 
-        if cli.args.clean:
-            commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
-        commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))
+        # If we got here, then we have a valid keyboard and keymap for a build target
+        target = KeyboardKeymapBuildTarget(cli.config.flash.keyboard, cli.config.flash.keymap)
 
-    if not commands:
+    if not target:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
         cli.print_help()
         return False
 
-    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
-    if not cli.args.dry_run:
-        cli.echo('\n')
-        for command in commands:
-            ret = cli.run(command, capture_output=False)
-            if ret.returncode:
-                return ret.returncode
+    target.configure(parallel=cli.config.flash.parallel, clean=cli.args.clean)
+    target.compile(cli.args.bootloader, dry_run=cli.args.dry_run, **envs)
diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py
index 3b9e150708b9..1598c2b67cca 100755
--- a/lib/python/qmk/cli/generate/compilation_database.py
+++ b/lib/python/qmk/cli/generate/compilation_database.py
@@ -12,7 +12,7 @@
 
 from milc import cli, MILC
 
-from qmk.commands import create_make_command, create_make_target
+from qmk.commands import create_make_target
 from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
@@ -79,7 +79,9 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
 def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None) -> bool:
     # Generate the make command for a specific keyboard/keymap.
     if not command:
-        command = create_make_command(keyboard, keymap, dry_run=True)
+        from qmk.build_targets import KeyboardKeymapBuildTarget  # Lazy load due to circular references
+        target = KeyboardKeymapBuildTarget(keyboard, keymap)
+        command = target.compile_command(dry_run=True)
 
     if not command:
         cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 06e6e411a72a..1f48318acb77 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -8,7 +8,7 @@
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE
-from qmk.commands import _find_make, get_make_parallel_args
+from qmk.commands import find_make, get_make_parallel_args
 from qmk.search import search_keymap_targets, search_make_targets
 
 
@@ -16,7 +16,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
     if len(targets) == 0:
         return
 
-    make_cmd = _find_make()
+    make_cmd = find_make()
     builddir = Path(QMK_FIRMWARE) / '.build'
     makefile = builddir / 'parallel_kb_builds.mk'
 
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 34696e37930c..7563499e9451 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -2,18 +2,16 @@
 """
 import os
 import sys
-import json
 import shutil
 from pathlib import Path
 
 from milc import cli
 import jsonschema
 
-from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX
 from qmk.json_schema import json_load, validate
 
 
-def _find_make():
+def find_make():
     """Returns the correct make command for this environment.
     """
     make_cmd = os.environ.get('MAKE')
@@ -46,7 +44,7 @@ def create_make_target(target, dry_run=False, parallel=1, **env_vars):
         A command that can be run to make the specified keyboard and keymap
     """
     env = []
-    make_cmd = _find_make()
+    make_cmd = find_make()
 
     for key, value in env_vars.items():
         env.append(f'{key}={value}')
@@ -57,41 +55,6 @@ def create_make_target(target, dry_run=False, parallel=1, **env_vars):
     return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]
 
 
-def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars):
-    """Create a make compile command
-
-    Args:
-
-        keyboard
-            The path of the keyboard, for example 'plank'
-
-        keymap
-            The name of the keymap, for example 'algernon'
-
-        target
-            Usually a bootloader.
-
-        dry_run
-            make -n -- don't actually build
-
-        parallel
-            The number of make jobs to run in parallel
-
-        **env_vars
-            Environment variables to be passed to make.
-
-    Returns:
-
-        A command that can be run to make the specified keyboard and keymap
-    """
-    make_args = [keyboard, keymap]
-
-    if target:
-        make_args.append(target)
-
-    return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars)
-
-
 def get_make_parallel_args(parallel=1):
     """Returns the arguments for running the specified number of parallel jobs.
     """
@@ -110,96 +73,6 @@ def get_make_parallel_args(parallel=1):
     return parallel_args
 
 
-def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars):
-    """Convert a configurator export JSON file into a C file and then compile it.
-
-    Args:
-
-        user_keymap
-            A deserialized keymap export
-
-        bootloader
-            A bootloader to flash
-
-        parallel
-            The number of make jobs to run in parallel
-
-    Returns:
-
-        A command to run to compile and flash the C file.
-    """
-    # In case the user passes a keymap.json from a keymap directory directly to the CLI.
-    # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
-    user_keymap["keymap"] = user_keymap.get("keymap", "default_json")
-
-    keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
-    target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
-    intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{keyboard_filesafe}_{user_keymap["keymap"]}')
-    keymap_dir = intermediate_output / 'src'
-    keymap_json = keymap_dir / 'keymap.json'
-
-    if clean:
-        if intermediate_output.exists():
-            shutil.rmtree(intermediate_output)
-
-    # begin with making the deepest folder in the tree
-    keymap_dir.mkdir(exist_ok=True, parents=True)
-
-    # Compare minified to ensure consistent comparison
-    new_content = json.dumps(user_keymap, separators=(',', ':'))
-    if keymap_json.exists():
-        old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
-        if old_content == new_content:
-            new_content = None
-
-    # Write the keymap.json file if different
-    if new_content:
-        keymap_json.write_text(new_content, encoding='utf-8')
-
-    # Return a command that can be run to make the keymap and flash if given
-    verbose = 'true' if cli.config.general.verbose else 'false'
-    color = 'true' if cli.config.general.color else 'false'
-    make_command = [_find_make()]
-
-    if not cli.config.general.verbose:
-        make_command.append('-s')
-
-    make_command.extend([
-        *get_make_parallel_args(parallel),
-        '-r',
-        '-R',
-        '-f',
-        'builddefs/build_keyboard.mk',
-    ])
-
-    if bootloader:
-        make_command.append(bootloader)
-
-    make_command.extend([
-        f'KEYBOARD={user_keymap["keyboard"]}',
-        f'KEYMAP={user_keymap["keymap"]}',
-        f'KEYBOARD_FILESAFE={keyboard_filesafe}',
-        f'TARGET={target}',
-        f'INTERMEDIATE_OUTPUT={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_1={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_2={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_3={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_4={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_5={intermediate_output}',
-        f'KEYMAP_JSON={keymap_json}',
-        f'KEYMAP_PATH={keymap_dir}',
-        f'VERBOSE={verbose}',
-        f'COLOR={color}',
-        'SILENT=false',
-        'QMK_BIN="qmk"',
-    ])
-
-    for key, value in env_vars.items():
-        make_command.append(f'{key}={value}')
-
-    return make_command
-
-
 def parse_configurator_json(configurator_file):
     """Open and parse a configurator json export
     """

From 09fdab9cae8212cf1affc4d3790112a41e08a1aa Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 9 Oct 2023 22:59:37 +1100
Subject: [PATCH 59/84] Refactoring.

---
 lib/python/qmk/build_targets.py               |  6 ++--
 lib/python/qmk/cli/clean.py                   |  4 +--
 lib/python/qmk/cli/flash.py                   | 10 ------
 .../qmk/cli/generate/compilation_database.py  |  4 +--
 lib/python/qmk/commands.py                    | 33 -------------------
 5 files changed, 8 insertions(+), 49 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 7097e25e3133..721892306075 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -6,7 +6,7 @@
 from pathlib import Path
 from milc import cli
 from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
-from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json, create_make_target
+from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
 from qmk.cli.generate.compilation_database import write_compilation_database
 
 
@@ -93,7 +93,9 @@ def generate_compilation_database(self, build_target: str = None, skip_clean: bo
 
     def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         if self._clean or self._compiledb:
-            command = create_make_target("clean", dry_run=dry_run, parallel=1, **env_vars)
+            command = [find_make(), "clean"]
+            if dry_run:
+                command.append('-n')
             cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command))
             cli.run(command, capture_output=False)
 
diff --git a/lib/python/qmk/cli/clean.py b/lib/python/qmk/cli/clean.py
index 72b7ffe81071..bdec01e4b663 100644
--- a/lib/python/qmk/cli/clean.py
+++ b/lib/python/qmk/cli/clean.py
@@ -2,7 +2,7 @@
 """
 from subprocess import DEVNULL
 
-from qmk.commands import create_make_target
+from qmk.commands import find_make
 from milc import cli
 
 
@@ -11,4 +11,4 @@
 def clean(cli):
     """Runs `make clean` (or `make distclean` if --all is passed)
     """
-    cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
+    cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL)
diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
index 89c01c280c50..5df31e4781bc 100644
--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -17,16 +17,6 @@
 from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
 
 
-def _is_keymap_target(keyboard, keymap):
-    if keymap == 'all':
-        return True
-
-    if locate_keymap(keyboard, keymap):
-        return True
-
-    return False
-
-
 def _list_bootloaders():
     """Prints the available bootloaders listed in docs.qmk.fm.
     """
diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py
index 1598c2b67cca..d25cf0af4af2 100755
--- a/lib/python/qmk/cli/generate/compilation_database.py
+++ b/lib/python/qmk/cli/generate/compilation_database.py
@@ -12,7 +12,7 @@
 
 from milc import cli, MILC
 
-from qmk.commands import create_make_target
+from qmk.commands import find_make
 from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
@@ -94,7 +94,7 @@ def write_compilation_database(keyboard: str = None, keymap: str = None, output_
 
     # re-use same executable as the main make invocation (might be gmake)
     if not skip_clean:
-        clean_command = create_make_target("clean")
+        clean_command = [find_make(), "clean"]
         cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
         cli.run(clean_command, capture_output=False, check=True, env=env)
 
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 7563499e9451..40cfc94ccfdc 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -22,39 +22,6 @@ def find_make():
     return make_cmd
 
 
-def create_make_target(target, dry_run=False, parallel=1, **env_vars):
-    """Create a make command
-
-    Args:
-
-        target
-            Usually a make rule, such as 'clean' or 'all'.
-
-        dry_run
-            make -n -- don't actually build
-
-        parallel
-            The number of make jobs to run in parallel
-
-        **env_vars
-            Environment variables to be passed to make.
-
-    Returns:
-
-        A command that can be run to make the specified keyboard and keymap
-    """
-    env = []
-    make_cmd = find_make()
-
-    for key, value in env_vars.items():
-        env.append(f'{key}={value}')
-
-    if cli.config.general.verbose:
-        env.append('VERBOSE=true')
-
-    return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]
-
-
 def get_make_parallel_args(parallel=1):
     """Returns the arguments for running the specified number of parallel jobs.
     """

From 9a1758fc557a8291c9bf9c34a2ece7cc66c927c0 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 9 Oct 2023 23:39:30 +1100
Subject: [PATCH 60/84] Refactoring.

---
 lib/python/qmk/cli/find.py         |  4 ++--
 lib/python/qmk/cli/mass_compile.py | 20 ++++++++++++--------
 lib/python/qmk/commands.py         |  2 +-
 lib/python/qmk/search.py           |  9 +++++----
 4 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index f2135bbc16c8..83dad6c7a5fa 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -24,8 +24,8 @@ def find(cli):
         cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
 
     targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
-    for keyboard, keymap, print_vals in targets:
-        print(f'{keyboard}:{keymap}')
+    for target, print_vals in targets:
+        print(f'{target}')
 
         for key, val in print_vals:
             print(f'    {key}={val}')
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 1f48318acb77..68779a2e0053 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -8,11 +8,11 @@
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE
-from qmk.commands import find_make, get_make_parallel_args
+from qmk.commands import find_make, get_make_parallel_args, build_environment
 from qmk.search import search_keymap_targets, search_make_targets
 
 
-def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
+def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, **env):
     if len(targets) == 0:
         return
 
@@ -22,7 +22,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
 
     if dry_run:
         cli.log.info('Compilation targets:')
-        for target in sorted(targets):
+        for target in sorted(targets, key=lambda t: f'{t}'):
             cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}")
     else:
         if clean:
@@ -30,9 +30,13 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
 
         builddir.mkdir(parents=True, exist_ok=True)
         with open(makefile, "w") as f:
-            for target in sorted(targets):
-                keyboard_name = target[0]
-                keymap_name = target[1]
+            for target, _ in sorted(targets, key=lambda t: f'{t[0]}'):
+                keyboard_name = target.keyboard
+                keymap_name = target.keymap
+                target.configure(parallel=1)  # We ignore parallelism on a per-build basis as we defer to the parent make invocation
+                target.prepare_build(**env)
+                command = target.compile_command(**env)
+                command[0] = '+@$(MAKE)'  # Override the make so that we can use jobserver to handle parallelism
                 keyboard_safe = keyboard_name.replace('/', '_')
                 build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
                 failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
@@ -43,7 +47,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
 {keyboard_safe}_{keymap_name}_binary:
 	@rm -f "{build_log}" || true
 	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(env)} \\
+	{' '.join(command)} \\
 		>>"{build_log}" 2>&1 \\
 		|| cp "{build_log}" "{failed_log}"
 	@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
@@ -99,4 +103,4 @@ def mass_compile(cli):
     else:
         targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
 
-    return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env)
+    return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, build_environment(cli.args.env))
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 40cfc94ccfdc..e5fdbc4f3003 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -30,7 +30,7 @@ def get_make_parallel_args(parallel=1):
     if int(parallel) <= 0:
         # 0 or -1 means -j without argument (unlimited jobs)
         parallel_args.append('--jobs')
-    else:
+    elif int(parallel) > 1:
         parallel_args.append('--jobs=' + str(parallel))
 
     if int(parallel) != 1:
diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 12b239ac6722..aaa41641ba31 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -13,6 +13,7 @@
 from qmk.info import keymap_json
 import qmk.keyboard
 import qmk.keymap
+from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget
 
 
 def _set_log_level(level):
@@ -181,13 +182,13 @@ def f(e):
     return targets
 
 
-def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[BuildTarget, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
+    return [(KeyboardKeymapBuildTarget(e[0], e[1]), e[2]) for e in sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1]))]
 
 
-def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[BuildTarget, List[Tuple[str, str]]]]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
+    return [(KeyboardKeymapBuildTarget(e[0], e[1]), e[2]) for e in sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1]))]

From a4fb79aa2d40e9959c42cf6a687887931914ec01 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 13 Oct 2023 09:36:40 +1100
Subject: [PATCH 61/84] Commentary.

---
 lib/python/qmk/cli/mass_compile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 68779a2e0053..8299dc373573 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -34,7 +34,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, **env):
                 keyboard_name = target.keyboard
                 keymap_name = target.keymap
                 target.configure(parallel=1)  # We ignore parallelism on a per-build basis as we defer to the parent make invocation
-                target.prepare_build(**env)
+                target.prepare_build(**env)  # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
                 command = target.compile_command(**env)
                 command[0] = '+@$(MAKE)'  # Override the make so that we can use jobserver to handle parallelism
                 keyboard_safe = keyboard_name.replace('/', '_')

From 931b10bbed1276f2feff8a54c5a1b41c400aefec Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 15 Oct 2023 12:03:24 +1100
Subject: [PATCH 62/84] `qmk pytest`

---
 lib/python/qmk/cli/doctor/main.py |  2 +-
 lib/python/qmk/cli/format/json.py | 75 +++++++++++++++++++------------
 lib/python/qmk/cli/new/keymap.py  |  2 -
 lib/python/qmk/commands.py        |  2 +-
 lib/python/qmk/userspace.py       |  2 +-
 5 files changed, 49 insertions(+), 34 deletions(-)

diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 69511f12b230..dd8b58b2c757 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -101,7 +101,7 @@ def userspace_tests(qmk_firmware):
         try:
             qmk_userspace_validate(path)
             cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`')
-        except FileNotFoundError as err:
+        except FileNotFoundError:
             cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Missing `qmk.json`')
         except UserspaceValidationError as err:
             cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`')
diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py
index 4f6b6d9e79d6..283513254c5c 100755
--- a/lib/python/qmk/cli/format/json.py
+++ b/lib/python/qmk/cli/format/json.py
@@ -13,32 +13,34 @@
 from qmk.path import normpath
 
 
-@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
-@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
-@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
-@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
-@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
-def format_json(cli):
-    """Format a json file.
+def _detect_json_format(file, json_data):
+    """Detect the format of a json file.
     """
-    json_file = json_load(cli.args.json_file)
+    json_encoder = None
+    try:
+        validate(json_data, 'qmk.user_repo.v1')
+        json_encoder = UserspaceJSONEncoder
+    except ValidationError:
+        pass
 
-    if cli.args.format == 'auto':
-        json_encoder = None
+    if json_encoder is None:
         try:
-            validate(json_file, 'qmk.user_repo.v1')
-            json_encoder = UserspaceJSONEncoder
+            validate(json_data, 'qmk.keyboard.v1')
+            json_encoder = InfoJSONEncoder
         except ValidationError as e:
-            pass
-
-        if json_encoder is None:
-            try:
-                validate(json_file, 'qmk.keyboard.v1')
-                json_encoder = InfoJSONEncoder
-            except ValidationError as e:
-                cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', cli.args.json_file, e)
-                cli.log.info('Treating %s as a keymap file.', cli.args.json_file)
-                json_encoder = KeymapJSONEncoder
+            cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', file, e)
+            cli.log.info('Treating %s as a keymap file.', file)
+            json_encoder = KeymapJSONEncoder
+
+    return json_encoder
+
+
+def _get_json_encoder(file, json_data):
+    """Get the json encoder for a file.
+    """
+    json_encoder = None
+    if cli.args.format == 'auto':
+        json_encoder = _detect_json_format(file, json_data)
     elif cli.args.format == 'keyboard':
         json_encoder = InfoJSONEncoder
     elif cli.args.format == 'keymap':
@@ -48,18 +50,33 @@ def format_json(cli):
     else:
         # This should be impossible
         cli.log.error('Unknown format: %s', cli.args.format)
+    return json_encoder
+
+
+@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
+@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
+@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
+@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
+@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
+def format_json(cli):
+    """Format a json file.
+    """
+    json_data = json_load(cli.args.json_file)
+
+    json_encoder = _get_json_encoder(cli.args.json_file, json_data)
+    if json_encoder is None:
         return False
 
-    if json_encoder == KeymapJSONEncoder and 'layout' in json_file:
+    if json_encoder == KeymapJSONEncoder and 'layout' in json_data:
         # Attempt to format the keycodes.
-        layout = json_file['layout']
-        info_data = info_json(json_file['keyboard'])
+        layout = json_data['layout']
+        info_data = info_json(json_data['keyboard'])
 
         if layout in info_data.get('layout_aliases', {}):
-            layout = json_file['layout'] = info_data['layout_aliases'][layout]
+            layout = json_data['layout'] = info_data['layout_aliases'][layout]
 
         if layout in info_data.get('layouts'):
-            for layer_num, layer in enumerate(json_file['layers']):
+            for layer_num, layer in enumerate(json_data['layers']):
                 current_layer = []
                 last_row = 0
 
@@ -70,9 +87,9 @@ def format_json(cli):
 
                     current_layer.append(keymap_key)
 
-                json_file['layers'][layer_num] = current_layer
+                json_data['layers'][layer_num] = current_layer
 
-    output = json.dumps(json_file, cls=json_encoder, sort_keys=True)
+    output = json.dumps(json_data, cls=json_encoder, sort_keys=True)
 
     if cli.args.inplace:
         with open(cli.args.json_file, 'w+', encoding='utf-8') as outfile:
diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index c2c1c975329b..9b0ac221a4ae 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -1,12 +1,10 @@
 """This script automates the copying of the default keymap into your own keymap.
 """
 import shutil
-from pathlib import Path
 
 from milc import cli
 from milc.questions import question
 
-from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE, QMK_FIRMWARE
 from qmk.path import is_keyboard, keymaps, keymap
 from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 31e6e9e703c0..e9c132433aeb 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -8,7 +8,7 @@
 from milc import cli
 import jsonschema
 
-from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX, QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.json_schema import json_load, validate
 
 
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index 71fef031ada5..fe50ceb0d118 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -121,4 +121,4 @@ def exceptions(self):
     def add(self, schema, exception):
         self.__exceptions.append((schema, exception))
         errorlist = "\n\n".join([f"{schema}: {exception}" for schema, exception in self.__exceptions])
-        self.message = f'Could not validate against any version of the userspace schema. Errors:\n\n{errorlist}'
\ No newline at end of file
+        self.message = f'Could not validate against any version of the userspace schema. Errors:\n\n{errorlist}'

From 4554eabd0b3e7c2283c80a8d4cceec6d189a3f43 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 16 Oct 2023 13:21:06 +1100
Subject: [PATCH 63/84] Fixup use of multiprocessing.

---
 lib/python/qmk/util.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py
index b1c4c7366b1a..7a485fef0936 100644
--- a/lib/python/qmk/util.py
+++ b/lib/python/qmk/util.py
@@ -5,7 +5,6 @@
 import multiprocessing
 
 from milc import cli
-from milc.subcommand import config
 
 
 @contextlib.contextmanager
@@ -23,13 +22,6 @@ def parallelize():
     else:
         parallel_search = cli.config.user.parallel_search
 
-    # If we haven't already written a value, write it to the file
-    if cli.config.user.parallel_search != parallel_search:
-        cli.args.read_only = False
-        config.set_config('user', 'parallel_search', parallel_search)
-        cli.save_config()
-        cli.config.user.parallel_search = parallel_search
-
     # Non-parallel searches use `map()`
     if not parallel_search:
         yield map
@@ -51,4 +43,8 @@ def parallel_map(*args, **kwargs):
     """Effectively runs `map()` but executes it in parallel if necessary.
     """
     with parallelize() as map_fn:
-        return map_fn(*args, **kwargs)
+        # This needs to be enclosed in a `list()` as some implementations return
+        # a generator function, which means the scope of the pool is closed off
+        # before the results are returned. Returning a list ensures results are
+        # materialised before any worker pool is shut down.
+        return list(map_fn(*args, **kwargs))

From 88b4c33fcde4794d1abe564dc79d75b13694b5f7 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 16 Oct 2023 13:55:05 +1100
Subject: [PATCH 64/84] Ensure consistent experience with `parallelize()`,
 regardless of backend.

---
 lib/python/qmk/search.py |  8 ++------
 lib/python/qmk/util.py   | 10 ++++++++--
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 12b239ac6722..a74450ca87d9 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -47,15 +47,11 @@ def _keymap_exists(keyboard, keymap):
         return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
 
 
-def _load_keymap_info(arg0, arg1=None):
+def _load_keymap_info(kb_km):
     """Returns a tuple of (keyboard, keymap, info.json) for the given keyboard/keymap combination.
-
-    Caters for the different unpacking requirements of each variant of map()/imap_unordered().
     """
     with ignore_logging():
-        if arg1 is None:
-            return (arg0[0], arg0[1], keymap_json(arg0[0], arg0[1]))
-        return (arg0, arg1, keymap_json(arg0, arg1))
+        return (kb_km[0], kb_km[1], keymap_json(kb_km[0], kb_km[1]))
 
 
 def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]:
diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py
index 7a485fef0936..db7debd5788f 100644
--- a/lib/python/qmk/util.py
+++ b/lib/python/qmk/util.py
@@ -1,7 +1,6 @@
 """Utility functions.
 """
 import contextlib
-import functools
 import multiprocessing
 
 from milc import cli
@@ -30,8 +29,15 @@ def parallelize():
     # Prefer mpire's `WorkerPool` if it's available
     with contextlib.suppress(ImportError):
         from mpire import WorkerPool
+        from mpire.utils import make_single_arguments
         with WorkerPool() as pool:
-            yield functools.partial(pool.imap_unordered, progress_bar=True)
+
+            def _worker(func, *args):
+                # Ensure we don't unpack tuples -- mpire's `WorkerPool` tries to do so normally so we tell it not to.
+                for r in pool.imap_unordered(func, make_single_arguments(*args, generator=False), progress_bar=True):
+                    yield r
+
+            yield _worker
         return
 
     # Otherwise fall back to multiprocessing's `Pool`

From 37fcb4e2edb47fdbb09ceeca0ae6bac5933d7b80 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Sun, 22 Oct 2023 20:39:50 +1100
Subject: [PATCH 65/84] Fixups.

---
 lib/python/qmk/cli/userspace/add.py    |  2 +-
 lib/python/qmk/cli/userspace/list.py   | 13 +++++---
 lib/python/qmk/cli/userspace/remove.py |  2 +-
 lib/python/qmk/keyboard.py             |  2 ++
 lib/python/qmk/userspace.py            | 42 +++++++++++++++++++++-----
 5 files changed, 48 insertions(+), 13 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index b440223303d1..e682c39fa5a8 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -20,4 +20,4 @@ def userspace_add(cli):
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
     userspace.add_target(cli.args.keyboard, cli.args.keymap)
-    userspace.save()
+    return userspace.save()
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index 7ace9140107d..f6f1fd640501 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -2,7 +2,7 @@
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.userspace import UserspaceDefs
-from qmk.keyboard import keyboard_folder_or_all, is_all_keyboards
+from qmk.keyboard import is_all_keyboards
 from qmk.keymap import is_keymap_target
 from qmk.search import search_keymap_targets
 
@@ -20,9 +20,14 @@ def userspace_list(cli):
     if cli.args.expand:
         build_targets = search_keymap_targets(build_targets)
 
-    for target in build_targets:
-        keyboard = keyboard_folder_or_all(target[0])
-        keymap = target[1]
+    for e in build_targets:
+        if isinstance(e[0], str):
+            keyboard = e[0]
+            keymap = e[1]
+        else:
+            keyboard = e[0].keyboard
+            keymap = e[0].keymap
+
         if is_all_keyboards(keyboard) or is_keymap_target(keyboard, keymap):
             cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
         else:
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
index d4b3e6387442..d0037debbfdd 100644
--- a/lib/python/qmk/cli/userspace/remove.py
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -16,4 +16,4 @@ def userspace_remove(cli):
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
     userspace.remove_target(cli.args.keyboard, cli.args.keymap)
-    userspace.save()
+    return userspace.save()
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 957de0b794b1..11b453502823 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -51,6 +51,8 @@ def __eq__(self, other):
 def is_all_keyboards(keyboard):
     """Returns True if the keyboard is an AllKeyboards object.
     """
+    if isinstance(keyboard, str):
+        return (keyboard == 'all')
     return isinstance(keyboard, AllKeyboards)
 
 
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index fe50ceb0d118..af2c71e6c749 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -19,20 +19,33 @@ def qmk_userspace_paths():
             test_dirs.append(current_dir)
         current_dir = current_dir.parent
 
-    test_dirs.append(environ.get('QMK_USERSPACE'))
-    test_dirs.append(cli.config.user.overlay_dir)
-    test_dirs = list(dict.fromkeys([Path(x) for x in filter(lambda x: x is not None and Path(x).is_dir(), test_dirs)]))
+    # If we have a QMK_USERSPACE environment variable, use that
+    if environ.get('QMK_USERSPACE') is not None:
+        current_dir = Path(environ.get('QMK_USERSPACE'))
+        if current_dir.is_dir():
+            test_dirs.append(current_dir)
+
+    # If someone has configured a directory, use that
+    if cli.config.user.overlay_dir is not None:
+        current_dir = Path(cli.config.user.overlay_dir)
+        if current_dir.is_dir():
+            test_dirs.append(current_dir)
+
     return test_dirs
 
 
 def qmk_userspace_validate(path):
+    # Construct a UserspaceDefs object to ensure it validates correctly
     if (path / 'qmk.json').is_file():
         UserspaceDefs(path / 'qmk.json')
         return
+
+    # No qmk.json file found
     raise FileNotFoundError('No qmk.json file found.')
 
 
 def detect_qmk_userspace():
+    # Iterate through all the detected userspace paths and return the first one that validates correctly
     test_dirs = qmk_userspace_paths()
     for test_dir in test_dirs:
         try:
@@ -87,23 +100,38 @@ def save(self):
             cli.log.error(f'Could not save userspace file: {err}')
             return False
 
-        self.path.write_text(json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True))
+        # Only actually write out data if it changed
+        old_data = json.dumps(json.loads(self.path.read_text()), cls=UserspaceJSONEncoder, sort_keys=True)
+        new_data = json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True)
+        if old_data != new_data:
+            self.path.write_text(new_data)
+            cli.log.info(f'Saved userspace file to {self.path}.')
         return True
 
-    def add_target(self, keyboard, keymap):
+    def add_target(self, keyboard, keymap, do_print=True):
         e = {"keyboard": keyboard, "keymap": keymap}
         if e not in self.build_targets:
             self.build_targets.append(e)
+            if do_print:
+                cli.log.info(f'Added {keyboard}:{keymap} to userspace build targets.')
+        else:
+            if do_print:
+                cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.')
 
-    def remove_target(self, keyboard, keymap):
+    def remove_target(self, keyboard, keymap, do_print=True):
         e = {"keyboard": keyboard, "keymap": keymap}
         if e in self.build_targets:
             self.build_targets.remove(e)
+            if do_print:
+                cli.log.info(f'Removed {keyboard}:{keymap} from userspace build targets.')
+        else:
+            if do_print:
+                cli.log.info(f'{keyboard}:{keymap} is not a userspace build target.')
 
     def __load_v1(self, json):
         for e in json['build_targets']:
             if len(e) == 2:
-                self.add_target(e[0], e[1])
+                self.add_target(e[0], e[1], False)
 
 
 class UserspaceValidationError(Exception):

From 57e507b72105f5f6fe1a9900ece38bba677bd8d2 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 30 Oct 2023 17:12:00 +1100
Subject: [PATCH 66/84] Fixup compiledb

---
 lib/python/qmk/cli/generate/compilation_database.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py
index d25cf0af4af2..5100d2b6d25c 100755
--- a/lib/python/qmk/cli/generate/compilation_database.py
+++ b/lib/python/qmk/cli/generate/compilation_database.py
@@ -76,12 +76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
     return records
 
 
-def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None) -> bool:
+def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool:
     # Generate the make command for a specific keyboard/keymap.
     if not command:
         from qmk.build_targets import KeyboardKeymapBuildTarget  # Lazy load due to circular references
         target = KeyboardKeymapBuildTarget(keyboard, keymap)
-        command = target.compile_command(dry_run=True)
+        command = target.compile_command(dry_run=True, **env_vars)
 
     if not command:
         cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')

From 8d2fdd1d0ea4f3d31692fe8520741bd376b1683c Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 30 Oct 2023 17:12:00 +1100
Subject: [PATCH 67/84] Fixup compiledb

---
 lib/python/qmk/cli/generate/compilation_database.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py
index d25cf0af4af2..5100d2b6d25c 100755
--- a/lib/python/qmk/cli/generate/compilation_database.py
+++ b/lib/python/qmk/cli/generate/compilation_database.py
@@ -76,12 +76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
     return records
 
 
-def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None) -> bool:
+def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool:
     # Generate the make command for a specific keyboard/keymap.
     if not command:
         from qmk.build_targets import KeyboardKeymapBuildTarget  # Lazy load due to circular references
         target = KeyboardKeymapBuildTarget(keyboard, keymap)
-        command = target.compile_command(dry_run=True)
+        command = target.compile_command(dry_run=True, **env_vars)
 
     if not command:
         cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')

From c50dcfadb53ce220dc1436f554d6bf308a8c6c2d Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Tue, 31 Oct 2023 19:11:16 +1100
Subject: [PATCH 68/84] Add support for json keymaps in `qmk mass-compile`

---
 lib/python/qmk/cli/mass_compile.py |  6 +++++-
 lib/python/qmk/keyboard.py         |  6 +-----
 lib/python/qmk/search.py           | 20 ++++++++++----------
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 7004a52632ff..2fc7a5f7839d 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -10,6 +10,7 @@
 from qmk.constants import QMK_FIRMWARE
 from qmk.commands import find_make, get_make_parallel_args, build_environment
 from qmk.search import search_keymap_targets, search_make_targets
+from qmk.build_targets import JsonKeymapBuildTarget
 
 
 def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, **env):
@@ -99,7 +100,10 @@ def mass_compile(cli):
     """Compile QMK Firmware against all keyboards.
     """
     if len(cli.args.builds) > 0:
-        targets = search_make_targets(cli.args.builds, cli.args.filter)
+        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+        targets = search_make_targets(make_like_targets, cli.args.filter)
+        targets.extend([(JsonKeymapBuildTarget(e), []) for e in json_like_targets])
     else:
         targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
 
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 9826f3f8876b..1aa63687d4ac 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -98,11 +98,7 @@ def keyboard_folder(keyboard):
         if keyboard == last_keyboard:
             break
 
-    rules_mk_file = Path(base_path, keyboard, 'rules.mk')
-
-    if rules_mk_file.exists():
-        rules_mk = parse_rules_mk_file(rules_mk_file)
-        keyboard = rules_mk.get('DEFAULT_FOLDER', keyboard)
+    keyboard = resolve_keyboard(keyboard)
 
     if not qmk.path.is_keyboard(keyboard):
         raise ValueError(f'Invalid keyboard: {keyboard}')
diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 8dd08e73c67c..47f20a960226 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -11,8 +11,8 @@
 
 from qmk.util import parallel_map
 from qmk.info import keymap_json
-import qmk.keyboard
-import qmk.keymap
+from qmk.keyboard import list_keyboards, keyboard_folder
+from qmk.keymap import list_keymaps, locate_keymap
 from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget
 
 
@@ -37,15 +37,15 @@ def _all_keymaps(keyboard):
     """Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard.
     """
     with ignore_logging():
-        keyboard = qmk.keyboard.resolve_keyboard(keyboard)
-        return [(keyboard, keymap) for keymap in qmk.keymap.list_keymaps(keyboard)]
+        keyboard = keyboard_folder(keyboard)
+        return [(keyboard, keymap) for keymap in list_keymaps(keyboard)]
 
 
 def _keymap_exists(keyboard, keymap):
     """Returns the keyboard name if the keyboard+keymap combination exists, otherwise None.
     """
     with ignore_logging():
-        return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
+        return keyboard if locate_keymap(keyboard, keymap) is not None else None
 
 
 def _load_keymap_info(kb_km):
@@ -76,7 +76,7 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
     Caters for 'all' in either keyboard or keymap, or both.
     """
     if all_keyboards is None:
-        all_keyboards = qmk.keyboard.list_keyboards()
+        all_keyboards = list_keyboards()
 
     if keyboard == 'all':
         if keymap == 'all':
@@ -88,21 +88,21 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
         else:
             cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...')
             keyboard_filter = functools.partial(_keymap_exists, keymap=keymap)
-            return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
+            return [(keyboard_folder(kb), keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
     else:
         if keymap == 'all':
-            keyboard = qmk.keyboard.resolve_keyboard(keyboard)
+            keyboard = keyboard_folder(keyboard)
             cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...')
             return _all_keymaps(keyboard)
         else:
-            return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)]
+            return [(keyboard_folder(keyboard), keymap)]
 
 
 def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
     """Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
     """
     overall_targets = []
-    all_keyboards = qmk.keyboard.list_keyboards()
+    all_keyboards = list_keyboards()
     for target in targets:
         overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
     return list(sorted(set(overall_targets)))

From 55820dac22afea9a303de5a3fbc985463fe7066c Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Tue, 31 Oct 2023 22:06:43 +1100
Subject: [PATCH 69/84] Initial support for json builds in userspace.

---
 data/schemas/definitions.jsonschema     | 18 ++++++++
 data/schemas/user_repo_v1.jsonschema    |  9 +---
 lib/python/qmk/build_targets.py         | 13 ++++++
 lib/python/qmk/cli/userspace/compile.py | 10 ++++-
 lib/python/qmk/cli/userspace/list.py    | 25 ++++++++---
 lib/python/qmk/userspace.py             | 57 ++++++++++++++++---------
 6 files changed, 97 insertions(+), 35 deletions(-)

diff --git a/data/schemas/definitions.jsonschema b/data/schemas/definitions.jsonschema
index 441e6395cf6e..ea29343d0ac0 100644
--- a/data/schemas/definitions.jsonschema
+++ b/data/schemas/definitions.jsonschema
@@ -177,5 +177,23 @@
         "type": "integer",
         "minimum": 0,
         "maximum": 1
+    },
+    "keyboard_keymap_tuple": {
+        "type": "array",
+        "prefixItems": [
+            { "$ref": "#/keyboard" },
+            { "$ref": "#/filename" }
+        ],
+        "unevaluatedItems": false
+    },
+    "json_file_path": {
+        "type": "string",
+        "pattern": "^[0-9a-z_/\\-]+\\.json$"
+    },
+    "build_target": {
+        "oneOf": [
+            { "$ref": "#/keyboard_keymap_tuple" },
+            { "$ref": "#/json_file_path" }
+        ]
     }
 }
diff --git a/data/schemas/user_repo_v1.jsonschema b/data/schemas/user_repo_v1.jsonschema
index 6a2efc2e9521..6cdf758685c5 100644
--- a/data/schemas/user_repo_v1.jsonschema
+++ b/data/schemas/user_repo_v1.jsonschema
@@ -15,14 +15,7 @@
         "build_targets": {
             "type": "array",
             "items": {
-                "type": "array",
-                "minItems": 2,
-                "maxItems": 2,
-                "items": {
-                    "type": "string",
-                    "minLength": 1,
-                    "pattern": "^[0-9a-z][0-9a-z_/]*$"
-                }
+                "$ref": "qmk.definitions.v1#/build_target"
             }
         }
     }
diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 721892306075..1d37f91e1ed0 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -8,6 +8,7 @@
 from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
 from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
 from qmk.cli.generate.compilation_database import write_compilation_database
+from qmk.keymap import locate_keymap
 
 
 class BuildTarget:
@@ -128,6 +129,18 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
         for key, value in env_vars.items():
             compile_args.append(f'{key}={value}')
 
+        # Need to override the keymap path just in case a userspace directory
+        # has their keymap located at a historical aliased path
+        keymap_location = locate_keymap(self.keyboard, self.keymap)
+        keymap_directory = keymap_location.parent
+        compile_args.extend([
+            f'MAIN_KEYMAP_PATH_1={keymap_directory}',
+            f'MAIN_KEYMAP_PATH_2={keymap_directory}',
+            f'MAIN_KEYMAP_PATH_3={keymap_directory}',
+            f'MAIN_KEYMAP_PATH_4={keymap_directory}',
+            f'MAIN_KEYMAP_PATH_5={keymap_directory}',
+        ])
+
         return compile_args
 
 
diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
index d21886e9080b..c1d8970ff3c9 100644
--- a/lib/python/qmk/cli/userspace/compile.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -1,8 +1,10 @@
+from pathlib import Path
 from milc import cli
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.commands import build_environment
 from qmk.userspace import UserspaceDefs
+from qmk.build_targets import JsonKeymapBuildTarget
 from qmk.search import search_keymap_targets
 from qmk.cli.mass_compile import mass_compile_targets
 
@@ -20,7 +22,11 @@ def userspace_compile(cli):
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
 
-    build_targets = [(e['keyboard'], e['keymap']) for e in userspace.build_targets]
-    build_targets = search_keymap_targets(build_targets)
+    build_targets = []
+    for e in userspace.build_targets:
+        if isinstance(e, Path):
+            build_targets.append([JsonKeymapBuildTarget(e), None])
+        elif isinstance(e, dict):
+            build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
 
     mass_compile_targets(build_targets, cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env))
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index f6f1fd640501..c7a775e487d1 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -1,3 +1,4 @@
+from pathlib import Path
 from milc import cli
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
@@ -16,15 +17,27 @@ def userspace_list(cli):
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
 
-    build_targets = [(e['keyboard'], e['keymap']) for e in userspace.build_targets]
     if cli.args.expand:
-        build_targets = search_keymap_targets(build_targets)
+        build_targets = []
+        for e in userspace.build_targets:
+            if isinstance(e, Path):
+                build_targets.append(e)
+            elif isinstance(e, dict):
+                build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
+    else:
+        build_targets = userspace.build_targets
 
     for e in build_targets:
-        if isinstance(e[0], str):
-            keyboard = e[0]
-            keymap = e[1]
-        else:
+        if isinstance(e, Path):
+            # JSON keymap from userspace
+            cli.log.info(f'JSON keymap: {{fg_cyan}}{e}{{fg_reset}}')
+            continue
+        elif isinstance(e, dict):
+            # keyboard/keymap dict from userspace
+            keyboard = e['keyboard']
+            keymap = e['keymap']
+        elif isinstance(e, tuple):
+            # BuildTarget from search_keymap_targets()
             keyboard = e[0].keyboard
             keymap = e[0].keymap
 
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index af2c71e6c749..a91052512d14 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -91,7 +91,10 @@ def save(self):
         }
 
         for e in self.build_targets:
-            target_json['build_targets'].append([e['keyboard'], e['keymap']])
+            if isinstance(e, dict):
+                target_json['build_targets'].append([e['keyboard'], e['keymap']])
+            elif isinstance(e, Path):
+                target_json['build_targets'].append(str(e.relative_to(self.path.parent)))
 
         try:
             # Ensure what we're writing validates against the latest version of the schema
@@ -108,30 +111,46 @@ def save(self):
             cli.log.info(f'Saved userspace file to {self.path}.')
         return True
 
-    def add_target(self, keyboard, keymap, do_print=True):
-        e = {"keyboard": keyboard, "keymap": keymap}
-        if e not in self.build_targets:
-            self.build_targets.append(e)
-            if do_print:
-                cli.log.info(f'Added {keyboard}:{keymap} to userspace build targets.')
+    def add_target(self, keyboard, keymap=None, do_print=True):
+        if keymap is None:
+            # Assume we're adding a json filename/path
+            json_path = Path(keyboard)
+            self.build_targets.append(json_path)
         else:
-            if do_print:
-                cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.')
-
-    def remove_target(self, keyboard, keymap, do_print=True):
-        e = {"keyboard": keyboard, "keymap": keymap}
-        if e in self.build_targets:
-            self.build_targets.remove(e)
-            if do_print:
-                cli.log.info(f'Removed {keyboard}:{keymap} from userspace build targets.')
+            # Both keyboard/keymap specified
+            e = {"keyboard": keyboard, "keymap": keymap}
+            if e not in self.build_targets:
+                self.build_targets.append(e)
+                if do_print:
+                    cli.log.info(f'Added {keyboard}:{keymap} to userspace build targets.')
+            else:
+                if do_print:
+                    cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.')
+
+    def remove_target(self, keyboard, keymap=None, do_print=True):
+        if keymap is None:
+            # Assume we're adding a json filename/path
+            json_path = Path(keyboard)
+            self.build_targets.remove(json_path)
         else:
-            if do_print:
-                cli.log.info(f'{keyboard}:{keymap} is not a userspace build target.')
+            # Both keyboard/keymap specified
+            e = {"keyboard": keyboard, "keymap": keymap}
+            if e in self.build_targets:
+                self.build_targets.remove(e)
+                if do_print:
+                    cli.log.info(f'Removed {keyboard}:{keymap} from userspace build targets.')
+            else:
+                if do_print:
+                    cli.log.info(f'{keyboard}:{keymap} is not a userspace build target.')
 
     def __load_v1(self, json):
         for e in json['build_targets']:
-            if len(e) == 2:
+            if isinstance(e, list) and len(e) == 2:
                 self.add_target(e[0], e[1], False)
+            if isinstance(e, str):
+                p = self.path.parent / e
+                if p.exists() and p.suffix == '.json':
+                    self.add_target(p)
 
 
 class UserspaceValidationError(Exception):

From 329d0ffdb7aee9c9b23695e34061b3281a8865ce Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 1 Nov 2023 11:11:24 +1100
Subject: [PATCH 70/84] API cleanup

---
 lib/python/qmk/build_targets.py    | 31 +++++++++++++++++++++++-------
 lib/python/qmk/cli/find.py         | 12 ++++--------
 lib/python/qmk/cli/mass_compile.py | 13 +++++++------
 lib/python/qmk/search.py           | 21 ++++++++++----------
 4 files changed, 45 insertions(+), 32 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 721892306075..387da0366b4e 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -2,16 +2,18 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 import json
 import shutil
-from typing import List
+from typing import List, Union
 from pathlib import Path
+from dotty_dict import dotty, Dotty
 from milc import cli
 from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
 from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
+from qmk.info import keymap_json
 from qmk.cli.generate.compilation_database import write_compilation_database
 
 
 class BuildTarget:
-    def __init__(self, keyboard: str, keymap: str):
+    def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
         self._keyboard = keyboard
         self._keyboard_safe = keyboard.replace('/', '_')
         self._keymap = keymap
@@ -21,6 +23,7 @@ def __init__(self, keyboard: str, keymap: str):
         self._target = f'{self._keyboard_safe}_{self.keymap}'
         self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
         self._generated_files_path = self._intermediate_output / 'src'
+        self._json = json if isinstance(json, Dotty) else dotty(json)
 
     def __str__(self):
         return f'{self.keyboard}:{self.keymap}'
@@ -44,6 +47,14 @@ def keyboard(self) -> str:
     def keymap(self) -> str:
         return self._keymap
 
+    @property
+    def json(self) -> dict:
+        if not self._json:
+            self._load_json()
+        if not self._json:
+            return dotty({})
+        return self._json
+
     def _common_make_args(self, dry_run: bool = False, build_target: str = None):
         compile_args = [
             find_make(),
@@ -113,12 +124,15 @@ def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -
 
 
 class KeyboardKeymapBuildTarget(BuildTarget):
-    def __init__(self, keyboard: str, keymap: str):
-        super().__init__(keyboard, keymap)
+    def __init__(self, keyboard: str, keymap: str, json: dict = None):
+        super().__init__(keyboard=keyboard, keymap=keymap, json=json)
 
     def __repr__(self):
         return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
 
+    def _load_json(self):
+        self._json = dotty(keymap_json(self.keyboard, self.keymap))
+
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         pass
 
@@ -138,19 +152,22 @@ def __init__(self, json_path):
         else:
             self.json_path = None
 
-        self.json = parse_configurator_json(json_path)  # Will load from stdin if provided
+        json = parse_configurator_json(json_path)  # Will load from stdin if provided
 
         # In case the user passes a keymap.json from a keymap directory directly to the CLI.
         # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
-        self.json["keymap"] = self.json.get("keymap", "default_json")
+        json["keymap"] = json.get("keymap", "default_json")
 
-        super().__init__(self.json['keyboard'], self.json['keymap'])
+        super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json)
 
         self._keymap_json = self._generated_files_path / 'keymap.json'
 
     def __repr__(self):
         return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
 
+    def _load_json(self):
+        pass  # Already loaded in constructor
+
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         if self._clean:
             if self._intermediate_output.exists():
diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index 83dad6c7a5fa..dab088d78ac1 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -19,13 +19,9 @@
 def find(cli):
     """Search through all keyboards and keymaps for a given search criteria.
     """
-
-    if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
-        cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
-
-    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
-    for target, print_vals in targets:
+    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter)
+    for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
         print(f'{target}')
 
-        for key, val in print_vals:
-            print(f'    {key}={val}')
+        for key in cli.args.print:
+            print(f'    {key}={target.json.get(key, None)}')
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 2fc7a5f7839d..7968de53e79a 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -3,6 +3,7 @@
 This will compile everything in parallel, for testing purposes.
 """
 import os
+from typing import List
 from pathlib import Path
 from subprocess import DEVNULL
 from milc import cli
@@ -10,10 +11,10 @@
 from qmk.constants import QMK_FIRMWARE
 from qmk.commands import find_make, get_make_parallel_args, build_environment
 from qmk.search import search_keymap_targets, search_make_targets
-from qmk.build_targets import JsonKeymapBuildTarget
+from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget
 
 
-def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, **env):
+def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env):
     if len(targets) == 0:
         return
 
@@ -23,7 +24,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, **env):
 
     if dry_run:
         cli.log.info('Compilation targets:')
-        for target in sorted(targets, key=lambda t: f'{t}'):
+        for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
             cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}")
     else:
         if clean:
@@ -31,7 +32,7 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, **env):
 
         builddir.mkdir(parents=True, exist_ok=True)
         with open(makefile, "w") as f:
-            for target, _ in sorted(targets, key=lambda t: f'{t[0]}'):
+            for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
                 keyboard_name = target.keyboard
                 keymap_name = target.keymap
                 target.configure(parallel=1)  # We ignore parallelism on a per-build basis as we defer to the parent make invocation
@@ -102,8 +103,8 @@ def mass_compile(cli):
     if len(cli.args.builds) > 0:
         json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
         make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
-        targets = search_make_targets(make_like_targets, cli.args.filter)
-        targets.extend([(JsonKeymapBuildTarget(e), []) for e in json_like_targets])
+        targets = search_make_targets(make_like_targets)
+        targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])
     else:
         targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
 
diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
index 47f20a960226..301d69c6c252 100644
--- a/lib/python/qmk/search.py
+++ b/lib/python/qmk/search.py
@@ -88,14 +88,13 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
         else:
             cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...')
             keyboard_filter = functools.partial(_keymap_exists, keymap=keymap)
-            return [(keyboard_folder(kb), keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
+            return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
     else:
         if keymap == 'all':
-            keyboard = keyboard_folder(keyboard)
             cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...')
             return _all_keymaps(keyboard)
         else:
-            return [(keyboard_folder(keyboard), keymap)]
+            return [(keyboard, keymap)]
 
 
 def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
@@ -108,13 +107,13 @@ def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str
     return list(sorted(set(overall_targets)))
 
 
-def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
     """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
 
     Optionally includes the values of the queried info.json keys.
     """
-    if len(filters) == 0 and len(print_vals) == 0:
-        targets = [(kb, km, {}) for kb, km in target_list]
+    if len(filters) == 0:
+        targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list]
     else:
         cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
         valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
@@ -173,18 +172,18 @@ def f(e):
                 cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
                 continue
 
-            targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps]
+            targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps]
 
     return targets
 
 
-def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[BuildTarget, List[Tuple[str, str]]]]:
+def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
     """Search for build targets matching the supplied criteria.
     """
-    return [(KeyboardKeymapBuildTarget(e[0], e[1]), e[2]) for e in sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1]))]
+    return _filter_keymap_targets(expand_keymap_targets(targets), filters)
 
 
-def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[BuildTarget, List[Tuple[str, str]]]]:
+def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]:
     """Search for build targets matching the supplied criteria.
     """
-    return [(KeyboardKeymapBuildTarget(e[0], e[1]), e[2]) for e in sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1]))]
+    return _filter_keymap_targets(expand_make_targets(targets), filters)

From d0d4d51737b6e0c1d92f96edefc89cc16379e20e Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 1 Nov 2023 11:18:59 +1100
Subject: [PATCH 71/84] API resync

---
 lib/python/qmk/cli/userspace/compile.py |  2 +-
 lib/python/qmk/cli/userspace/list.py    | 12 +++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
index c1d8970ff3c9..19d86e05bcf3 100644
--- a/lib/python/qmk/cli/userspace/compile.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -25,7 +25,7 @@ def userspace_compile(cli):
     build_targets = []
     for e in userspace.build_targets:
         if isinstance(e, Path):
-            build_targets.append([JsonKeymapBuildTarget(e), None])
+            build_targets.append(JsonKeymapBuildTarget(e))
         elif isinstance(e, dict):
             build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
 
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index c7a775e487d1..e511c9057a97 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -1,8 +1,10 @@
 from pathlib import Path
+from dotty_dict import Dotty
 from milc import cli
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.userspace import UserspaceDefs
+from qmk.build_targets import BuildTarget
 from qmk.keyboard import is_all_keyboards
 from qmk.keymap import is_keymap_target
 from qmk.search import search_keymap_targets
@@ -22,7 +24,7 @@ def userspace_list(cli):
         for e in userspace.build_targets:
             if isinstance(e, Path):
                 build_targets.append(e)
-            elif isinstance(e, dict):
+            elif isinstance(e, dict) or isinstance(e, Dotty):
                 build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
     else:
         build_targets = userspace.build_targets
@@ -32,14 +34,14 @@ def userspace_list(cli):
             # JSON keymap from userspace
             cli.log.info(f'JSON keymap: {{fg_cyan}}{e}{{fg_reset}}')
             continue
-        elif isinstance(e, dict):
+        elif isinstance(e, dict) or isinstance(e, Dotty):
             # keyboard/keymap dict from userspace
             keyboard = e['keyboard']
             keymap = e['keymap']
-        elif isinstance(e, tuple):
+        elif isinstance(e, BuildTarget):
             # BuildTarget from search_keymap_targets()
-            keyboard = e[0].keyboard
-            keymap = e[0].keymap
+            keyboard = e.keyboard
+            keymap = e.keymap
 
         if is_all_keyboards(keyboard) or is_keymap_target(keyboard, keymap):
             cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')

From 34df02877f6ec92fc5e84e0e0e8ddb731e2f2466 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 1 Nov 2023 11:26:21 +1100
Subject: [PATCH 72/84] Deal with dotty_dict.

---
 lib/python/qmk/build_targets.py | 10 +++++++---
 lib/python/qmk/cli/find.py      |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 387da0366b4e..a1912419c4be 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -23,7 +23,7 @@ def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
         self._target = f'{self._keyboard_safe}_{self.keymap}'
         self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
         self._generated_files_path = self._intermediate_output / 'src'
-        self._json = json if isinstance(json, Dotty) else dotty(json)
+        self._json = json.to_dict() if isinstance(json, Dotty) else json
 
     def __str__(self):
         return f'{self.keyboard}:{self.keymap}'
@@ -52,9 +52,13 @@ def json(self) -> dict:
         if not self._json:
             self._load_json()
         if not self._json:
-            return dotty({})
+            return {}
         return self._json
 
+    @property
+    def dotty(self) -> Dotty:
+        return dotty(self.json)
+
     def _common_make_args(self, dry_run: bool = False, build_target: str = None):
         compile_args = [
             find_make(),
@@ -131,7 +135,7 @@ def __repr__(self):
         return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
 
     def _load_json(self):
-        self._json = dotty(keymap_json(self.keyboard, self.keymap))
+        self._json = keymap_json(self.keyboard, self.keymap)
 
     def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
         pass
diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py
index dab088d78ac1..55a053009248 100644
--- a/lib/python/qmk/cli/find.py
+++ b/lib/python/qmk/cli/find.py
@@ -24,4 +24,4 @@ def find(cli):
         print(f'{target}')
 
         for key in cli.args.print:
-            print(f'    {key}={target.json.get(key, None)}')
+            print(f'    {key}={target.dotty.get(key, None)}')

From 3104b430857ed57f8a7d3853de29a62ecea2b7f1 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 1 Nov 2023 11:59:42 +1100
Subject: [PATCH 73/84] Userspace add/remove.

---
 lib/python/qmk/cli/userspace/add.py    | 26 +++++++++++++++++++++-----
 lib/python/qmk/cli/userspace/remove.py | 18 +++++++++++++++++-
 lib/python/qmk/userspace.py            | 23 ++++++++++-------------
 3 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index e682c39fa5a8..10fa7f22e2ed 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -1,3 +1,4 @@
+from pathlib import Path
 from milc import cli
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
@@ -6,6 +7,7 @@
 from qmk.userspace import UserspaceDefs
 
 
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap> to compile in parallel. Specifying this overrides all other target search options.")
 @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.subcommand('Adds a build target to userspace `qmk.json`.')
@@ -14,10 +16,24 @@ def userspace_add(cli):
         cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
         return False
 
-    if not is_keymap_target(cli.args.keyboard, cli.args.keymap):
-        cli.log.error('Invalid keymap argument.')
-        return False
-
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
-    userspace.add_target(cli.args.keyboard, cli.args.keymap)
+
+    if len(cli.args.builds) > 0:
+        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+
+        for e in json_like_targets:
+            userspace.add_target(json_path=e)
+
+        for e in make_like_targets:
+            s = e.split(':')
+            userspace.add_target(keyboard=s[0], keymap=s[1])
+
+    else:
+        if not is_keymap_target(cli.args.keyboard, cli.args.keymap):
+            cli.log.error('Invalid keymap argument.')
+            return False
+
+        userspace.add_target(cli.args.keyboard, cli.args.keymap)
+
     return userspace.save()
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
index d0037debbfdd..80b7986c949d 100644
--- a/lib/python/qmk/cli/userspace/remove.py
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -1,3 +1,4 @@
+from pathlib import Path
 from milc import cli
 
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
@@ -6,6 +7,7 @@
 from qmk.userspace import UserspaceDefs
 
 
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap> to compile in parallel. Specifying this overrides all other target search options.")
 @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.subcommand('Removes a build target from userspace `qmk.json`.')
@@ -15,5 +17,19 @@ def userspace_remove(cli):
         return False
 
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
-    userspace.remove_target(cli.args.keyboard, cli.args.keymap)
+
+    if len(cli.args.builds) > 0:
+        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+
+        for e in json_like_targets:
+            userspace.remove_target(json_path=e)
+
+        for e in make_like_targets:
+            s = e.split(':')
+            userspace.remove_target(keyboard=s[0], keymap=s[1])
+
+    else:
+        userspace.remove_target(cli.args.keyboard, cli.args.keymap)
+
     return userspace.save()
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index a91052512d14..59d75eafa62c 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -111,12 +111,11 @@ def save(self):
             cli.log.info(f'Saved userspace file to {self.path}.')
         return True
 
-    def add_target(self, keyboard, keymap=None, do_print=True):
-        if keymap is None:
+    def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
+        if json_path is not None:
             # Assume we're adding a json filename/path
-            json_path = Path(keyboard)
-            self.build_targets.append(json_path)
-        else:
+            self.build_targets.append(Path(json_path))
+        elif keyboard is not None and keymap is not None:
             # Both keyboard/keymap specified
             e = {"keyboard": keyboard, "keymap": keymap}
             if e not in self.build_targets:
@@ -127,12 +126,10 @@ def add_target(self, keyboard, keymap=None, do_print=True):
                 if do_print:
                     cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.')
 
-    def remove_target(self, keyboard, keymap=None, do_print=True):
-        if keymap is None:
-            # Assume we're adding a json filename/path
-            json_path = Path(keyboard)
-            self.build_targets.remove(json_path)
-        else:
+    def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
+        if json_path is not None:
+            self.build_targets.remove(Path(json_path))
+        elif keyboard is not None and keymap is not None:
             # Both keyboard/keymap specified
             e = {"keyboard": keyboard, "keymap": keymap}
             if e in self.build_targets:
@@ -146,11 +143,11 @@ def remove_target(self, keyboard, keymap=None, do_print=True):
     def __load_v1(self, json):
         for e in json['build_targets']:
             if isinstance(e, list) and len(e) == 2:
-                self.add_target(e[0], e[1], False)
+                self.add_target(keyboard=e[0], keymap=e[1], do_print=False)
             if isinstance(e, str):
                 p = self.path.parent / e
                 if p.exists() and p.suffix == '.json':
-                    self.add_target(p)
+                    self.add_target(json_path=p, do_print=False)
 
 
 class UserspaceValidationError(Exception):

From 2ab8e76d85079708d6196dbef5773eea4848a113 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 1 Nov 2023 12:13:14 +1100
Subject: [PATCH 74/84] Command help, actually add the json files.

---
 lib/python/qmk/cli/userspace/add.py    | 4 ++--
 lib/python/qmk/cli/userspace/remove.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index 10fa7f22e2ed..e86551625535 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -7,7 +7,7 @@
 from qmk.userspace import UserspaceDefs
 
 
-@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap> to compile in parallel. Specifying this overrides all other target search options.")
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
 @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.subcommand('Adds a build target to userspace `qmk.json`.')
@@ -34,6 +34,6 @@ def userspace_add(cli):
             cli.log.error('Invalid keymap argument.')
             return False
 
-        userspace.add_target(cli.args.keyboard, cli.args.keymap)
+        userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
 
     return userspace.save()
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
index 80b7986c949d..45c720c3b818 100644
--- a/lib/python/qmk/cli/userspace/remove.py
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -7,7 +7,7 @@
 from qmk.userspace import UserspaceDefs
 
 
-@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap> to compile in parallel. Specifying this overrides all other target search options.")
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
 @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.subcommand('Removes a build target from userspace `qmk.json`.')
@@ -30,6 +30,6 @@ def userspace_remove(cli):
             userspace.remove_target(keyboard=s[0], keymap=s[1])
 
     else:
-        userspace.remove_target(cli.args.keyboard, cli.args.keymap)
+        userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
 
     return userspace.save()

From 5c48e581d99821f81f095feff65bfd8a729b0492 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 1 Nov 2023 12:58:59 +1100
Subject: [PATCH 75/84] Fixup list.

---
 lib/python/qmk/cli/userspace/list.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index e511c9057a97..4e50046b89c4 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -5,7 +5,7 @@
 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.userspace import UserspaceDefs
 from qmk.build_targets import BuildTarget
-from qmk.keyboard import is_all_keyboards
+from qmk.keyboard import is_all_keyboards, keyboard_folder
 from qmk.keymap import is_keymap_target
 from qmk.search import search_keymap_targets
 
@@ -43,7 +43,7 @@ def userspace_list(cli):
             keyboard = e.keyboard
             keymap = e.keymap
 
-        if is_all_keyboards(keyboard) or is_keymap_target(keyboard, keymap):
+        if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap):
             cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
         else:
             cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!')

From 45f89dd68ad76889b9893a13ac3b85ccc266fe77 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 1 Nov 2023 13:10:45 +1100
Subject: [PATCH 76/84] Resolve aliases.

---
 lib/python/qmk/build_targets.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index a1912419c4be..236c2eaa037b 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -8,14 +8,15 @@
 from milc import cli
 from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
 from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
+from qmk.keyboard import keyboard_folder
 from qmk.info import keymap_json
 from qmk.cli.generate.compilation_database import write_compilation_database
 
 
 class BuildTarget:
     def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
-        self._keyboard = keyboard
-        self._keyboard_safe = keyboard.replace('/', '_')
+        self._keyboard = keyboard_folder(keyboard)
+        self._keyboard_safe = self._keyboard.replace('/', '_')
         self._keymap = keymap
         self._parallel = 1
         self._clean = False

From 7a4542a0f2e8278d0cd22b78b26bb3e954d42ea6 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 1 Nov 2023 13:15:52 +1100
Subject: [PATCH 77/84] License headers.

---
 lib/python/qmk/cli/userspace/add.py     | 2 ++
 lib/python/qmk/cli/userspace/compile.py | 2 ++
 lib/python/qmk/cli/userspace/doctor.py  | 2 ++
 lib/python/qmk/cli/userspace/list.py    | 2 ++
 lib/python/qmk/cli/userspace/remove.py  | 2 ++
 lib/python/qmk/userspace.py             | 2 ++
 6 files changed, 12 insertions(+)

diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index e86551625535..f6daec9c7807 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -1,3 +1,5 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
 from pathlib import Path
 from milc import cli
 
diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
index 19d86e05bcf3..14fb0d7b53a0 100644
--- a/lib/python/qmk/cli/userspace/compile.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -1,3 +1,5 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
 from pathlib import Path
 from milc import cli
 
diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py
index 8f6a08161e6e..2b7e29aa7eed 100644
--- a/lib/python/qmk/cli/userspace/doctor.py
+++ b/lib/python/qmk/cli/userspace/doctor.py
@@ -1,3 +1,5 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
index 4e50046b89c4..a63f669dd7b8 100644
--- a/lib/python/qmk/cli/userspace/list.py
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -1,3 +1,5 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
 from pathlib import Path
 from dotty_dict import Dotty
 from milc import cli
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
index 45c720c3b818..c7d180bfd123 100644
--- a/lib/python/qmk/cli/userspace/remove.py
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -1,3 +1,5 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
 from pathlib import Path
 from milc import cli
 
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index 59d75eafa62c..48e244c4e461 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -1,3 +1,5 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
 from os import environ
 from pathlib import Path
 import json

From 01bb7c29285437e0b61a502f1078adc2c1a8397d Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Fri, 17 Nov 2023 21:36:05 +1100
Subject: [PATCH 78/84] Prevent insertion of same json file multiple times.

---
 lib/python/qmk/build_targets.py | 23 +++++++++++++----------
 lib/python/qmk/path.py          | 25 ++++++++++++++++++++++++-
 lib/python/qmk/userspace.py     | 19 +++++++++++++++++--
 3 files changed, 54 insertions(+), 13 deletions(-)

diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 1782f12de0e1..8b1c81daa94f 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -12,6 +12,7 @@
 from qmk.info import keymap_json
 from qmk.cli.generate.compilation_database import write_compilation_database
 from qmk.keymap import locate_keymap
+from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace
 
 
 class BuildTarget:
@@ -148,17 +149,19 @@ def compile_command(self, build_target: str = None, dry_run: bool = False, **env
         for key, value in env_vars.items():
             compile_args.append(f'{key}={value}')
 
-        # Need to override the keymap path just in case a userspace directory
-        # has their keymap located at a historical aliased path
+        # Need to override the keymap path if the keymap is a userspace directory.
+        # This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap
+        # in an equivalent historical location.
         keymap_location = locate_keymap(self.keyboard, self.keymap)
-        keymap_directory = keymap_location.parent
-        compile_args.extend([
-            f'MAIN_KEYMAP_PATH_1={keymap_directory}',
-            f'MAIN_KEYMAP_PATH_2={keymap_directory}',
-            f'MAIN_KEYMAP_PATH_3={keymap_directory}',
-            f'MAIN_KEYMAP_PATH_4={keymap_directory}',
-            f'MAIN_KEYMAP_PATH_5={keymap_directory}',
-        ])
+        if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location):
+            keymap_directory = keymap_location.parent
+            compile_args.extend([
+                f'MAIN_KEYMAP_PATH_1={keymap_directory}',
+                f'MAIN_KEYMAP_PATH_2={keymap_directory}',
+                f'MAIN_KEYMAP_PATH_3={keymap_directory}',
+                f'MAIN_KEYMAP_PATH_4={keymap_directory}',
+                f'MAIN_KEYMAP_PATH_5={keymap_directory}',
+            ])
 
         return compile_args
 
diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py
index 67e5cced79fb..74364ee04b02 100644
--- a/lib/python/qmk/path.py
+++ b/lib/python/qmk/path.py
@@ -29,7 +29,7 @@ def under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])):
 
 
 def under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])):
-    """Returns a Path object representing the relative path under $QMK_USERSPACE, qmk root, or None.
+    """Returns a Path object representing the relative path under $QMK_USERSPACE, or None.
     """
     try:
         if HAS_QMK_USERSPACE:
@@ -39,6 +39,29 @@ def under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])):
     return None
 
 
+def is_under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])):
+    """Returns a boolean if the input path is a child under qmk_firmware.
+    """
+    if path is None:
+        return False
+    try:
+        return Path(os.path.commonpath([Path(path), QMK_FIRMWARE])) == QMK_FIRMWARE
+    except ValueError:
+        return False
+
+
+def is_under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])):
+    """Returns a boolean if the input path is a child under $QMK_USERSPACE.
+    """
+    if path is None:
+        return False
+    try:
+        if HAS_QMK_USERSPACE:
+            return Path(os.path.commonpath([Path(path), QMK_USERSPACE])) == QMK_USERSPACE
+    except ValueError:
+        return False
+
+
 def keyboard(keyboard_name):
     """Returns the path to a keyboard's directory relative to the qmk root.
     """
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
index 48e244c4e461..378356800699 100644
--- a/lib/python/qmk/userspace.py
+++ b/lib/python/qmk/userspace.py
@@ -116,7 +116,14 @@ def save(self):
     def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
         if json_path is not None:
             # Assume we're adding a json filename/path
-            self.build_targets.append(Path(json_path))
+            json_path = Path(json_path)
+            if json_path not in self.build_targets:
+                self.build_targets.append(json_path)
+                if do_print:
+                    cli.log.info(f'Added {json_path} to userspace build targets.')
+            else:
+                cli.log.info(f'{json_path} is already a userspace build target.')
+
         elif keyboard is not None and keymap is not None:
             # Both keyboard/keymap specified
             e = {"keyboard": keyboard, "keymap": keymap}
@@ -130,7 +137,15 @@ def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
 
     def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
         if json_path is not None:
-            self.build_targets.remove(Path(json_path))
+            # Assume we're removing a json filename/path
+            json_path = Path(json_path)
+            if json_path in self.build_targets:
+                self.build_targets.remove(json_path)
+                if do_print:
+                    cli.log.info(f'Removed {json_path} from userspace build targets.')
+            else:
+                cli.log.info(f'{json_path} is not a userspace build target.')
+
         elif keyboard is not None and keymap is not None:
             # Both keyboard/keymap specified
             e = {"keyboard": keyboard, "keymap": keymap}

From bdee41ff185122d3c3ae250e9eaac79bbaaa74b3 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 22 Nov 2023 14:31:05 +1100
Subject: [PATCH 79/84] Speed improvements.

---
 lib/python/qmk/cli/userspace/compile.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
index 14fb0d7b53a0..0a42dd5bf5bf 100644
--- a/lib/python/qmk/cli/userspace/compile.py
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -25,10 +25,14 @@ def userspace_compile(cli):
     userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
 
     build_targets = []
+    keyboard_keymap_targets = []
     for e in userspace.build_targets:
         if isinstance(e, Path):
             build_targets.append(JsonKeymapBuildTarget(e))
         elif isinstance(e, dict):
-            build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
+            keyboard_keymap_targets.append((e['keyboard'], e['keymap']))
 
-    mass_compile_targets(build_targets, cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env))
+    if len(keyboard_keymap_targets) > 0:
+        build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
+
+    mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env))

From 3cc6de8bf29ed7f2f1e77a7d07ea130d87d28529 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 22 Nov 2023 15:46:12 +1100
Subject: [PATCH 80/84] Automatically invoke `qmk new-keymap` if running `qmk
 userspace-add` with nonexistent keyboard/keymap combo.

---
 lib/python/qmk/cli/userspace/add.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
index f6daec9c7807..8993d54dba52 100644
--- a/lib/python/qmk/cli/userspace/add.py
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -32,10 +32,20 @@ def userspace_add(cli):
             userspace.add_target(keyboard=s[0], keymap=s[1])
 
     else:
-        if not is_keymap_target(cli.args.keyboard, cli.args.keymap):
-            cli.log.error('Invalid keymap argument.')
-            return False
-
-        userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
+        failed = False
+        try:
+            if not is_keymap_target(cli.args.keyboard, cli.args.keymap):
+                failed = True
+        except KeyError:
+            failed = True
+
+        if failed:
+            from qmk.cli.new.keymap import new_keymap
+            cli.config.new_keymap.keyboard = cli.args.keyboard
+            cli.config.new_keymap.keymap = cli.args.keymap
+            if new_keymap(cli) is not False:
+                userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
+        else:
+            userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
 
     return userspace.save()

From f4cd6900522510152f34e9060ab71375ae6128dc Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Wed, 22 Nov 2023 16:17:58 +1100
Subject: [PATCH 81/84] Fixup VPATH when userspace is present. Auto-add build
 target with `qmk new-keymap`.

---
 builddefs/build_keyboard.mk      | 5 +++++
 lib/python/qmk/cli/new/keymap.py | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/builddefs/build_keyboard.mk b/builddefs/build_keyboard.mk
index eb97fd62a3b3..f17171fe209b 100644
--- a/builddefs/build_keyboard.mk
+++ b/builddefs/build_keyboard.mk
@@ -390,6 +390,11 @@ ifeq ("$(USER_NAME)","")
 endif
 USER_PATH := users/$(USER_NAME)
 
+# If we have userspace, then add it to the lookup VPATH
+ifneq ($(wildcard $(QMK_USERSPACE)),)
+    VPATH += $(QMK_USERSPACE)
+endif
+
 # If the equivalent users directory exists in userspace, use that in preference to anything currently in the main repo
 ifneq ($(wildcard $(QMK_USERSPACE)/$(USER_PATH)),)
     USER_PATH := $(QMK_USERSPACE)/$(USER_PATH)
diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index 9b0ac221a4ae..d4339bc9ef00 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -5,10 +5,12 @@
 from milc import cli
 from milc.questions import question
 
+from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE
 from qmk.path import is_keyboard, keymaps, keymap
 from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.userspace import UserspaceDefs
 
 
 def prompt_keyboard():
@@ -68,3 +70,9 @@ def new_keymap(cli):
     # end message to user
     cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')
     cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.")
+
+    # Add to userspace compile if we have userspace available
+    if HAS_QMK_USERSPACE:
+        userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+        userspace.add_target(keyboard=kb_name, keymap=user_name, do_print=False)
+        return userspace.save()

From 71a2a047479f78db66ccda8f7f7396e97d102931 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 27 Nov 2023 00:56:04 +1100
Subject: [PATCH 82/84] Initial docs.

---
 docs/_summary.md                 |  2 +-
 docs/newbs_external_userspace.md | 87 ++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 docs/newbs_external_userspace.md

diff --git a/docs/_summary.md b/docs/_summary.md
index 722c5f9c5dd6..36c90c5bb9f6 100644
--- a/docs/_summary.md
+++ b/docs/_summary.md
@@ -4,7 +4,7 @@
   * [Building Your First Firmware](newbs_building_firmware.md)
   * [Flashing Firmware](newbs_flashing.md)
   * [Getting Help/Support](support.md)
-  * [Building With GitHub Userspace](newbs_building_firmware_workflow.md)
+  * [External Userspace](newbs_external_userspace.md)
   * [Other Resources](newbs_learn_more_resources.md)
   * [Syllabus](syllabus.md)
 
diff --git a/docs/newbs_external_userspace.md b/docs/newbs_external_userspace.md
new file mode 100644
index 000000000000..a9aec4ceee65
--- /dev/null
+++ b/docs/newbs_external_userspace.md
@@ -0,0 +1,87 @@
+# External QMK Userspace
+
+QMK Firmware can now be built using an external userspace git repository, optionally coupled with GitHub actions so that users don't need to build firmware locally.
+
+Using external userspace allows you to create your firmware without having to modify your QMK installation. Additionally, it drastically simplifies the process of keeping up to date with QMK Firmware, especially when leveraging GitHub actions to automatically perform builds for you.
+
+!> External QMK Userspace is new functionality and may have issues. Tighter integration with the `qmk` command will occur over time.
+
+?> Legacy GitHub-based firmware build instructions can be found [here](newbs_building_firmware_workflow.md). This document supersedes those instructions, but they should still correctly function.
+
+## Repository Setup (on GitHub)
+
+A basic skeleton External Userspace repository can be found [here](https://github.com/qmk/qmk_userspace). If you with to keep your keymaps on GitHub (strongly recommended!), you can fork the repository and use it as a base:
+
+![Userspace Fork](https://i.imgur.com/hcegguh.png)
+
+Going ahead with your fork will copy it to your account, at which point you can clone it to your local machine and begin adding your keymaps:
+
+![Userspace Clone](https://i.imgur.com/CWYmsk8.png)
+
+```sh
+git clone https://github.com/myusername/qmk_userspace.git
+qmk config user.overlay_dir="$(realpath qmk_userspace)"
+```
+
+## Repository Setup (local-only)
+
+If you don't want to use GitHub and prefer to keep everything local, you can clone a copy of the default userspace repository locally instead:
+
+```sh
+git clone https://github.com/qmk/qmk_userspace.git
+qmk config user.overlay_dir="$(realpath qmk_userspace)"
+```
+
+## Setting up QMK Locally
+
+If you wish to build on your local machine, you will need to set up QMK locally. This is a one-time process, and is documented in the [QMK Docs](https://docs.qmk.fm/#/newbs).
+
+!> If you want to use any QMK CLI commands related to manipulating userspace definitions, you will currently need a copy of QMK Firmware as well.
+
+!> Building locally has a much shorter turnaround time than waiting for GitHub actions to complete.
+
+## Adding a Keymap
+
+_These instructions assume you have already set up QMK locally, and have a copy of the QMK Firmware repository on your machine._
+
+Keymaps within external userspace are defined in the same way as they are in the main QMK repository. You can either use the `qmk new-keymap` command to create a new keymap, or manually create a new directory in the `keyboards` directory.
+
+After creating your new keymap, building the keymap matches normal QMK usage:
+
+```sh
+qmk compile -kb <keyboard> -km <keymap>
+```
+
+!> The `qmk config user.overlay_dir=...` command must have been run when cloning the userspace repository for this to work correctly.
+
+## Adding the keymap to userspace build targets
+
+Once you have created your keymap, you will need to add it to the userspace build targets. This is done using the `qmk userspace-add` command:
+
+```sh
+# for a keyboard/keymap combo:
+qmk userspace-add -kb <keyboard> -km <keymap>
+# or, for a json-based keymap (if kept "loose"):
+qmk userspace-add <relative/path/to/my/keymap.json>
+```
+
+This updates the `qmk.json` file in the root of your userspace repository.
+
+## Compiling userspace build targets
+
+Once you have added your keymaps to the userspace build targets, you can compile all of them at once using the `qmk userspace-compile` command:
+
+```sh
+qmk userspace-compile
+```
+
+## Using GitHub Actions
+
+GitHub Actions can be used to automatically build your keymaps whenever you push changes to your repository. If you have set up your list of build targets, this is as simple as enabling workflows in the GitHub repository settings:
+
+
+![Repo Settings](https://i.imgur.com/EVkxOt1.png)
+
+Any push will result in a build occurring, and once complete a new release containing the newly-minted firmware files:
+
+![Releases](https://i.imgur.com/zmwOL5P.png)

From 53f90cea75a13300ddc30e492c0a2bbdb4dbe3f5 Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 27 Nov 2023 22:49:57 +1100
Subject: [PATCH 83/84] Docs updates.

---
 docs/newbs_external_userspace.md | 63 ++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 27 deletions(-)

diff --git a/docs/newbs_external_userspace.md b/docs/newbs_external_userspace.md
index a9aec4ceee65..9bdf4b0b185c 100644
--- a/docs/newbs_external_userspace.md
+++ b/docs/newbs_external_userspace.md
@@ -1,16 +1,28 @@
 # External QMK Userspace
 
-QMK Firmware can now be built using an external userspace git repository, optionally coupled with GitHub actions so that users don't need to build firmware locally.
+QMK Firmware now officially supports storing user keymaps outside of the normal QMK Firmware repository, allowing users to maintain their own keymaps without having to fork, modify, and maintain a copy of QMK Firmware themselves.
 
-Using external userspace allows you to create your firmware without having to modify your QMK installation. Additionally, it drastically simplifies the process of keeping up to date with QMK Firmware, especially when leveraging GitHub actions to automatically perform builds for you.
+External Userspace mirrors the structure of the main QMK Firmware repository, but only contains the keymaps that you wish to build. You can still use `keyboards/<my keyboard>/keymaps/<my keymap>` to store your keymaps, or you can use the `layouts/<my layout>/<my keymap>` system as before -- they're just stored external to QMK Firmware.
 
-!> External QMK Userspace is new functionality and may have issues. Tighter integration with the `qmk` command will occur over time.
+The build system will still honor the use of `users/<my keymap>` if you rely on the traditional QMK Firmware [userspace feature](feature_userspace.md) -- it's now supported externally too, using the same location inside the External Userspace directory.
 
-?> Legacy GitHub-based firmware build instructions can be found [here](newbs_building_firmware_workflow.md). This document supersedes those instructions, but they should still correctly function.
+Additionally, there is first-class support for using GitHub Actions to build your keymaps, allowing you to automatically compile your keymaps whenever you push changes to your External Userspace repository.
 
-## Repository Setup (on GitHub)
+!> External Userspace is new functionality and may have issues. Tighter integration with the `qmk` command will occur over time.
 
-A basic skeleton External Userspace repository can be found [here](https://github.com/qmk/qmk_userspace). If you with to keep your keymaps on GitHub (strongly recommended!), you can fork the repository and use it as a base:
+?> Historical keymap.json and GitHub-based firmware build instructions can be found [here](newbs_building_firmware_workflow.md). This document supersedes those instructions, but they should still function correctly.
+
+## Setting up QMK Locally
+
+If you wish to build on your local machine, you will need to set up QMK locally. This is a one-time process, and is documented in the [newbs setup guide](https://docs.qmk.fm/#/newbs).
+
+!> If you wish to use any QMK CLI commands related to manipulating External Userspace definitions, you will currently need a copy of QMK Firmware as well.
+
+!> Building locally has a much shorter turnaround time than waiting for GitHub Actions to complete.
+
+## External Userspace Repository Setup (forked on GitHub)
+
+A basic skeleton External Userspace repository can be found [here](https://github.com/qmk/qmk_userspace). If you wish to keep your keymaps on GitHub (strongly recommended!), you can fork the repository and use it as a base:
 
 ![Userspace Fork](https://i.imgur.com/hcegguh.png)
 
@@ -19,32 +31,28 @@ Going ahead with your fork will copy it to your account, at which point you can
 ![Userspace Clone](https://i.imgur.com/CWYmsk8.png)
 
 ```sh
-git clone https://github.com/myusername/qmk_userspace.git
+cd $HOME
+git clone https://github.com/{myusername}/qmk_userspace.git
 qmk config user.overlay_dir="$(realpath qmk_userspace)"
 ```
 
-## Repository Setup (local-only)
+## External Userspace Setup (locally stored only)
 
-If you don't want to use GitHub and prefer to keep everything local, you can clone a copy of the default userspace repository locally instead:
+If you don't want to use GitHub and prefer to keep everything local, you can clone a copy of the default External Userspace locally instead:
 
 ```sh
+cd $HOME
 git clone https://github.com/qmk/qmk_userspace.git
 qmk config user.overlay_dir="$(realpath qmk_userspace)"
 ```
 
-## Setting up QMK Locally
-
-If you wish to build on your local machine, you will need to set up QMK locally. This is a one-time process, and is documented in the [QMK Docs](https://docs.qmk.fm/#/newbs).
-
-!> If you want to use any QMK CLI commands related to manipulating userspace definitions, you will currently need a copy of QMK Firmware as well.
-
-!> Building locally has a much shorter turnaround time than waiting for GitHub actions to complete.
-
 ## Adding a Keymap
 
 _These instructions assume you have already set up QMK locally, and have a copy of the QMK Firmware repository on your machine._
 
-Keymaps within external userspace are defined in the same way as they are in the main QMK repository. You can either use the `qmk new-keymap` command to create a new keymap, or manually create a new directory in the `keyboards` directory.
+Keymaps within External Userspace are defined in the same way as they are in the main QMK repository. You can either use the `qmk new-keymap` command to create a new keymap, or manually create a new directory in the `keyboards` directory.
+
+Alternatively, you can use the `layouts` directory to store your keymaps, using the same layout system as the main QMK repository -- if you choose to do so you'll want to use the path `layouts/<layout name>/<keymap name>/keymap.*` to store your keymap files, where `layout name` matches an existing layout in QMK, such as `tkl_ansi`.
 
 After creating your new keymap, building the keymap matches normal QMK usage:
 
@@ -52,11 +60,11 @@ After creating your new keymap, building the keymap matches normal QMK usage:
 qmk compile -kb <keyboard> -km <keymap>
 ```
 
-!> The `qmk config user.overlay_dir=...` command must have been run when cloning the userspace repository for this to work correctly.
+!> The `qmk config user.overlay_dir=...` command must have been run when cloning the External Userspace repository for this to work correctly.
 
-## Adding the keymap to userspace build targets
+## Adding the keymap to External Userspace build targets
 
-Once you have created your keymap, you will need to add it to the userspace build targets. This is done using the `qmk userspace-add` command:
+Once you have created your keymap, if you want to use GitHub Actions to build your firmware, you will need to add it to the External Userspace build targets. This is done using the `qmk userspace-add` command:
 
 ```sh
 # for a keyboard/keymap combo:
@@ -65,23 +73,24 @@ qmk userspace-add -kb <keyboard> -km <keymap>
 qmk userspace-add <relative/path/to/my/keymap.json>
 ```
 
-This updates the `qmk.json` file in the root of your userspace repository.
+This updates the `qmk.json` file in the root of your External Userspace directory. If you're using a git repository to store your keymaps, now is a great time to commit and push to your own fork.
 
-## Compiling userspace build targets
+## Compiling External Userspace build targets
 
-Once you have added your keymaps to the userspace build targets, you can compile all of them at once using the `qmk userspace-compile` command:
+Once you have added your keymaps to the External Userspace build targets, you can compile all of them at once using the `qmk userspace-compile` command:
 
 ```sh
 qmk userspace-compile
 ```
 
-## Using GitHub Actions
+All firmware builds you've added to the External Userspace build targets will be built, and the resulting firmware files will be placed in the root of your External Userspace directory.
 
-GitHub Actions can be used to automatically build your keymaps whenever you push changes to your repository. If you have set up your list of build targets, this is as simple as enabling workflows in the GitHub repository settings:
+## Using GitHub Actions
 
+GitHub Actions can be used to automatically build your keymaps whenever you push changes to your External Userspace repository. If you have set up your list of build targets, this is as simple as enabling workflows in the GitHub repository settings:
 
 ![Repo Settings](https://i.imgur.com/EVkxOt1.png)
 
-Any push will result in a build occurring, and once complete a new release containing the newly-minted firmware files:
+Any push will result in compilation of all configured builds, and once completed a new release containing the newly-minted firmware files will be created on GitHub, which you can subsequently download and flash to your keyboard:
 
 ![Releases](https://i.imgur.com/zmwOL5P.png)

From 4d745e1593e9da1d752bb353e32c0a71d48e7cbd Mon Sep 17 00:00:00 2001
From: Nick Brassel <nick@tzarc.org>
Date: Mon, 27 Nov 2023 23:00:53 +1100
Subject: [PATCH 84/84] Docs updates.

---
 docs/cli_commands.md | 125 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 125 insertions(+)

diff --git a/docs/cli_commands.md b/docs/cli_commands.md
index 79fd9de57576..7b5ad5b13ac7 100644
--- a/docs/cli_commands.md
+++ b/docs/cli_commands.md
@@ -482,6 +482,131 @@ $ qmk import-kbfirmware ~/Downloads/gh62.json
 
 ---
 
+# External Userspace Commands
+
+## `qmk userspace-add`
+
+This command adds a keyboard/keymap to the External Userspace build targets.
+
+**Usage**:
+
+```
+qmk userspace-add [-h] [-km KEYMAP] [-kb KEYBOARD] [builds ...]
+
+positional arguments:
+  builds                List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.
+
+options:
+  -h, --help            show this help message and exit
+  -km KEYMAP, --keymap KEYMAP
+                        The keymap to build a firmware for. Ignored when a configurator export is supplied.
+  -kb KEYBOARD, --keyboard KEYBOARD
+                        The keyboard to build a firmware for. Ignored when a configurator export is supplied.
+```
+
+**Example**:
+
+```
+$ qmk userspace-add -kb planck/rev6 -km default
+Ψ Added planck/rev6:default to userspace build targets
+Ψ Saved userspace file to /home/you/qmk_userspace/qmk.json
+```
+
+## `qmk userspace-remove`
+
+This command removes a keyboard/keymap from the External Userspace build targets.
+
+**Usage**:
+
+```
+qmk userspace-remove [-h] [-km KEYMAP] [-kb KEYBOARD] [builds ...]
+
+positional arguments:
+  builds                List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.
+
+options:
+  -h, --help            show this help message and exit
+  -km KEYMAP, --keymap KEYMAP
+                        The keymap to build a firmware for. Ignored when a configurator export is supplied.
+  -kb KEYBOARD, --keyboard KEYBOARD
+                        The keyboard to build a firmware for. Ignored when a configurator export is supplied.
+```
+
+**Example**:
+
+```
+$ qmk userspace-remove -kb planck/rev6 -km default
+Ψ Removed planck/rev6:default from userspace build targets
+Ψ Saved userspace file to /home/you/qmk_userspace/qmk.json
+```
+
+## `qmk userspace-list`
+
+This command lists the External Userspace build targets.
+
+**Usage**:
+
+```
+qmk userspace-list [-h] [-e]
+
+options:
+  -h, --help    show this help message and exit
+  -e, --expand  Expands any use of `all` for either keyboard or keymap.
+```
+
+**Example**:
+
+```
+$ qmk userspace-list
+Ψ Current userspace build targets:
+Ψ Keyboard: planck/rev6, keymap: you
+Ψ Keyboard: clueboard/66/rev3, keymap: you
+```
+
+## `qmk userspace-compile`
+
+This command compiles all the External Userspace build targets.
+
+**Usage**:
+
+```
+qmk userspace-compile [-h] [-e ENV] [-n] [-c] [-j PARALLEL] [-t]
+
+options:
+  -h, --help            show this help message and exit
+  -e ENV, --env ENV     Set a variable to be passed to make. May be passed multiple times.
+  -n, --dry-run         Don't actually build, just show the commands to be run.
+  -c, --clean           Remove object files before compiling.
+  -j PARALLEL, --parallel PARALLEL
+                        Set the number of parallel make jobs; 0 means unlimited.
+  -t, --no-temp         Remove temporary files during build.
+```
+
+**Example**:
+
+```
+$ qmk userspace-compile
+Ψ Preparing target list...
+Build planck/rev6:you                                                  [OK]
+Build clueboard/66/rev3:you                                            [OK]
+```
+
+## `qmk userspace-doctor`
+
+This command examines your environment and alerts you to potential problems related to External Userspace.
+
+**Example**:
+
+```
+% qmk userspace-doctor
+Ψ QMK home: /home/you/qmk_userspace/qmk_firmware
+Ψ Testing userspace candidate: /home/you/qmk_userspace -- Valid `qmk.json`
+Ψ QMK userspace: /home/you/qmk_userspace
+Ψ Userspace enabled: True
+```
+
+---
+
 # Developer Commands
 
 ## `qmk format-text`