From 47365aed32f878a8674acb0434a3dd95883e396e Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 13:40:16 -0700 Subject: [PATCH 01/34] pulled source from dev branch --- .gitignore | 4 ++ lib/python/qmk/cli/compile.py | 36 +++++++++++++--- lib/python/qmk/compile_commands_json.py | 56 +++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100755 lib/python/qmk/compile_commands_json.py diff --git a/.gitignore b/.gitignore index 20437224f184..951873b8fbdf 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,7 @@ __pycache__ # prerequisites for updating ChibiOS /util/fmpp* + +# clangd +compile_commands.json +.clangd/ diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 6480d624b043..cbbbb6d288f4 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -10,12 +10,16 @@ import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json +from qmk.compile_commands_json import parse_make_n, qmk_dir +from pathlib import Path +import json @cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export to compile') @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") +@cli.argument('-b', '--builddb', arg_only=True, action='store_true', help="Does a make clean, then a make -n, and uses the dry-run output to create a compilation database (compile_commands.json). This file can help some IDEs and IDE-like editors work better. For more information about this: https://clang.llvm.org/docs/JSONCompilationDatabase.html") @cli.subcommand('Compile a QMK Firmware.') @automagic_keyboard @automagic_keymap @@ -40,19 +44,39 @@ def compile(cli): else: if cli.config.compile.keyboard and cli.config.compile.keymap: # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap) + dry_run = cli.args.builddb + command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=dry_run) elif not cli.config.compile.keyboard: cli.log.error('Could not determine keyboard!') elif not cli.config.compile.keymap: cli.log.error('Could not determine keymap!') - # Compile the firmware, if we're able to if command: - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - subprocess.run(command) + if cli.args.builddb: + cli.log.info('Making clean') + subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + cli.log.info('Gathering build instructions') + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + db = parse_make_n(proc.stdout) + res = proc.wait() + if res != 0: + raise RuntimeError(f"Got error from: {repr(command)}") + + cli.log.info(f"Found {len(db)} compile commands") + + dbpath = qmk_dir / 'compile_commands.json' + + cli.log.info(f"Writing build database to {dbpath}") + dbpath.write_text(json.dumps(db, indent=4)) + + else: + # Compile the firmware, if we're able to + cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) + if not cli.args.dry_run: + cli.echo('\n') + subprocess.run(command) else: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py new file mode 100755 index 000000000000..4aae872351bf --- /dev/null +++ b/lib/python/qmk/compile_commands_json.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +import sys +import re +import inspect +from pathlib import Path +import json +import shlex +from itertools import takewhile +from functools import lru_cache +from subprocess import check_output + + +from typing import TextIO, List, Dict + +qmk_dir = Path(__file__).parent.parent.parent.parent + +@lru_cache(maxsize=10) +def system_libs(binary: str): + try: + return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include")) + except Exception: + return [] + + +file_re = re.compile(r"""printf "Compiling: ([^"]+)""") +cmd_re = re.compile(r"""LOG=\$\(([^\)]+)\)""") + +def parse_make_n(f: TextIO) -> List[Dict[str,str]]: + state = 'start' + this_file = None + records = [] + for line in f: + if state == 'start': + m = file_re.search(line) + if m: + this_file = m.group(1) + state = 'cmd' + + if state == 'cmd': + m = cmd_re.search(line) + if m: + # we have a hit! + this_cmd = m.group(1) + args = shlex.split(this_cmd) + args = list(takewhile(lambda x: x != '&&', args)) + args += ['-I%s' % s for s in system_libs(args[0])] + new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') + records.append({"directory": str(qmk_dir), "command": new_cmd, "file": this_file}) + state = 'start' + + return records + +if __name__ == '__main__': + with open(sys.argv[1]) as f: + print(json.dumps(parse_make_n(f), indent=4)) From e0bff4d788962bed6c8b6236e934091a6a897aff Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 13:52:20 -0700 Subject: [PATCH 02/34] missed a file from origin --- lib/python/qmk/commands.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 5d2a03c9a8aa..7572175caf43 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -10,7 +10,7 @@ import qmk.keymap -def create_make_command(keyboard, keymap, target=None): +def create_make_command(keyboard, keymap, target=None, dry_run=False): """Create a make compile command Args: @@ -24,6 +24,9 @@ def create_make_command(keyboard, keymap, target=None): target Usually a bootloader. + dry_run + make -n -- don't actually build + Returns: A command that can be run to make the specified keyboard and keymap @@ -34,7 +37,10 @@ def create_make_command(keyboard, keymap, target=None): if target: make_args.append(target) - return [make_cmd, ':'.join(make_args)] + if dry_run: + return [make_cmd, '-n', ':'.join(make_args)] + else: + return [make_cmd, ':'.join(make_args)] def compile_configurator_json(user_keymap, bootloader=None): From 5a51eaabfb03913b315ca284ab82eaa5435dfa74 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 13:57:16 -0700 Subject: [PATCH 03/34] formatting --- lib/python/qmk/cli/compile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index cbbbb6d288f4..c06f97a02b21 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -19,7 +19,11 @@ @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-b', '--builddb', arg_only=True, action='store_true', help="Does a make clean, then a make -n, and uses the dry-run output to create a compilation database (compile_commands.json). This file can help some IDEs and IDE-like editors work better. For more information about this: https://clang.llvm.org/docs/JSONCompilationDatabase.html") +@cli.argument('-b', '--builddb', arg_only=True, action='store_true', + help="Does a make clean, then a make -n for this target and uses the dry-run output to create " + "a compilation database (compile_commands.json). This file can help some IDEs and " + "IDE-like editors work better. For more information about this: " + "https://clang.llvm.org/docs/JSONCompilationDatabase.html") @cli.subcommand('Compile a QMK Firmware.') @automagic_keyboard @automagic_keymap From d817b7d0c3d7287c748a73d608b9380e30fce6c9 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 14:18:02 -0700 Subject: [PATCH 04/34] revised argument names. relaxed matching rules to work for avr too --- lib/python/qmk/cli/compile.py | 6 +++--- lib/python/qmk/compile_commands_json.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index c06f97a02b21..5f730dd2605c 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -19,7 +19,7 @@ @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-b', '--builddb', arg_only=True, action='store_true', +@cli.argument('-c', '--compile-commands', arg_only=True, action='store_true', help="Does a make clean, then a make -n for this target and uses the dry-run output to create " "a compilation database (compile_commands.json). This file can help some IDEs and " "IDE-like editors work better. For more information about this: " @@ -48,7 +48,7 @@ def compile(cli): else: if cli.config.compile.keyboard and cli.config.compile.keymap: # Generate the make command for a specific keyboard/keymap. - dry_run = cli.args.builddb + dry_run = cli.args.compile_commands command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=dry_run) elif not cli.config.compile.keyboard: @@ -57,7 +57,7 @@ def compile(cli): cli.log.error('Could not determine keymap!') if command: - if cli.args.builddb: + if cli.args.compile_commands: cli.log.info('Making clean') subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index 4aae872351bf..c6777cfcabaf 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -24,7 +24,7 @@ def system_libs(binary: str): file_re = re.compile(r"""printf "Compiling: ([^"]+)""") -cmd_re = re.compile(r"""LOG=\$\(([^\)]+)\)""") +cmd_re = re.compile(r"""LOG=\$\((.+)\&\&""") def parse_make_n(f: TextIO) -> List[Dict[str,str]]: state = 'start' @@ -43,7 +43,6 @@ def parse_make_n(f: TextIO) -> List[Dict[str,str]]: # we have a hit! this_cmd = m.group(1) args = shlex.split(this_cmd) - args = list(takewhile(lambda x: x != '&&', args)) args += ['-I%s' % s for s in system_libs(args[0])] new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') records.append({"directory": str(qmk_dir), "command": new_cmd, "file": this_file}) From 30b564f47c6078bbf138e0d42ad88c22b6922bb3 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 14:23:56 -0700 Subject: [PATCH 05/34] add docstrings --- lib/python/qmk/compile_commands_json.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index c6777cfcabaf..462be000af9f 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -15,8 +15,14 @@ qmk_dir = Path(__file__).parent.parent.parent.parent + @lru_cache(maxsize=10) def system_libs(binary: str): + """Find the system include directory that the given build tool uses. + + Only tested on OSX+homebrew so far. + """ + try: return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include")) except Exception: @@ -26,7 +32,14 @@ def system_libs(binary: str): file_re = re.compile(r"""printf "Compiling: ([^"]+)""") cmd_re = re.compile(r"""LOG=\$\((.+)\&\&""") -def parse_make_n(f: TextIO) -> List[Dict[str,str]]: + +def parse_make_n(f: TextIO) -> List[Dict[str, str]]: + """parse the output of `make -n ` + + This function makes many assumptions about the format of your build log. + This happens to work right now for qmk. + """ + state = 'start' this_file = None records = [] @@ -50,6 +63,7 @@ def parse_make_n(f: TextIO) -> List[Dict[str,str]]: return records + if __name__ == '__main__': with open(sys.argv[1]) as f: print(json.dumps(parse_make_n(f), indent=4)) From 849a12f741d8732a564ddcbe780da0809e023edf Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 14:41:33 -0700 Subject: [PATCH 06/34] added docs. tightened up regex --- docs/cli_commands.md | 17 +++++++++++++++++ lib/python/qmk/compile_commands_json.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index eff5321bdb15..52d59a13517a 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -71,6 +71,23 @@ $ qmk compile -kb dz60 ... ``` +**Creating a `compile_commands.json`** + +Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you. + +**Example:** + +``` +$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak +$ qmk compile -c +Ψ Making clean +Ψ Gathering build instructions +Ψ Found 50 compile commands +Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json +``` + +Now open your dev environment and live a squiggly-free life. + ## `qmk flash` This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default. diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index 462be000af9f..d18979e6aab4 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -30,7 +30,7 @@ def system_libs(binary: str): file_re = re.compile(r"""printf "Compiling: ([^"]+)""") -cmd_re = re.compile(r"""LOG=\$\((.+)\&\&""") +cmd_re = re.compile(r"""LOG=\$\((.+?)\&\&""") def parse_make_n(f: TextIO) -> List[Dict[str, str]]: From b671c40e5e6d3134532e57fc29f22bc8c985f60f Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 14:52:51 -0700 Subject: [PATCH 07/34] remove unused imports --- lib/python/qmk/cli/compile.py | 1 - lib/python/qmk/compile_commands_json.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 5f730dd2605c..70563832b9f5 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -11,7 +11,6 @@ from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json from qmk.compile_commands_json import parse_make_n, qmk_dir -from pathlib import Path import json diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index d18979e6aab4..e3ac188e96e7 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -2,11 +2,9 @@ import sys import re -import inspect from pathlib import Path import json import shlex -from itertools import takewhile from functools import lru_cache from subprocess import check_output From f42cadc361f02faf13ed954eb83b9a987d29578a Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 14:13:58 -0700 Subject: [PATCH 08/34] cleaning up command file. use existing qmk dir constant --- lib/python/qmk/cli/compile.py | 39 +++------------- lib/python/qmk/cli/compiledb.py | 61 +++++++++++++++++++++++++ lib/python/qmk/compile_commands_json.py | 5 +- 3 files changed, 69 insertions(+), 36 deletions(-) create mode 100755 lib/python/qmk/cli/compiledb.py diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 70563832b9f5..6480d624b043 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -10,19 +10,12 @@ import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json -from qmk.compile_commands_json import parse_make_n, qmk_dir -import json @cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export to compile') @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-c', '--compile-commands', arg_only=True, action='store_true', - help="Does a make clean, then a make -n for this target and uses the dry-run output to create " - "a compilation database (compile_commands.json). This file can help some IDEs and " - "IDE-like editors work better. For more information about this: " - "https://clang.llvm.org/docs/JSONCompilationDatabase.html") @cli.subcommand('Compile a QMK Firmware.') @automagic_keyboard @automagic_keymap @@ -47,39 +40,19 @@ def compile(cli): else: if cli.config.compile.keyboard and cli.config.compile.keymap: # Generate the make command for a specific keyboard/keymap. - dry_run = cli.args.compile_commands - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=dry_run) + command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap) elif not cli.config.compile.keyboard: cli.log.error('Could not determine keyboard!') elif not cli.config.compile.keymap: cli.log.error('Could not determine keymap!') + # Compile the firmware, if we're able to if command: - if cli.args.compile_commands: - cli.log.info('Making clean') - subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - cli.log.info('Gathering build instructions') - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - db = parse_make_n(proc.stdout) - res = proc.wait() - if res != 0: - raise RuntimeError(f"Got error from: {repr(command)}") - - cli.log.info(f"Found {len(db)} compile commands") - - dbpath = qmk_dir / 'compile_commands.json' - - cli.log.info(f"Writing build database to {dbpath}") - dbpath.write_text(json.dumps(db, indent=4)) - - else: - # Compile the firmware, if we're able to - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - subprocess.run(command) + cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) + if not cli.args.dry_run: + cli.echo('\n') + subprocess.run(command) else: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py new file mode 100755 index 000000000000..0878161eae22 --- /dev/null +++ b/lib/python/qmk/cli/compiledb.py @@ -0,0 +1,61 @@ +"""Creates a compilation database for the given keyboard build. +""" +import subprocess +from argparse import FileType + +from milc import cli + +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.commands import create_make_command +from qmk.compile_commands_json import parse_make_n +from qmk.constants import QMK_FIRMWARE +import json + + +@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.subcommand('Create a compilation database.') +@automagic_keyboard +@automagic_keymap +def compile(cli): + """Creates a compilation database for the given keyboard build. + + Does a make clean, then a make -n for this target and uses the dry-run output to create + a compilation database (compile_commands.json). This file can help some IDEs and + IDE-like editors work better. For more information about this: + + https://clang.llvm.org/docs/JSONCompilationDatabase.html + """ + command = None + + if cli.config.compile.keyboard and cli.config.compile.keymap: + # Generate the make command for a specific keyboard/keymap. + command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=True) + + elif not cli.config.compile.keyboard: + cli.log.error('Could not determine keyboard!') + elif not cli.config.compile.keymap: + cli.log.error('Could not determine keymap!') + + if command: + cli.log.info('Making clean') + subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + cli.log.info('Gathering build instructions') + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + db = parse_make_n(proc.stdout) + res = proc.wait() + if res != 0: + raise RuntimeError(f"Got error from: {repr(command)}") + + cli.log.info(f"Found {len(db)} compile commands") + + dbpath = QMK_FIRMWARE / 'compile_commands.json' + + cli.log.info(f"Writing build database to {dbpath}") + dbpath.write_text(json.dumps(db, indent=4)) + + else: + cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') + cli.echo('usage: qmk compiledb [-kb KEYBOARD] [-km KEYMAP]') + return False diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index e3ac188e96e7..2d54dc180c33 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -8,11 +8,10 @@ from functools import lru_cache from subprocess import check_output +from qmk.constants import QMK_FIRMWARE from typing import TextIO, List, Dict -qmk_dir = Path(__file__).parent.parent.parent.parent - @lru_cache(maxsize=10) def system_libs(binary: str): @@ -56,7 +55,7 @@ def parse_make_n(f: TextIO) -> List[Dict[str, str]]: args = shlex.split(this_cmd) args += ['-I%s' % s for s in system_libs(args[0])] new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') - records.append({"directory": str(qmk_dir), "command": new_cmd, "file": this_file}) + records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) state = 'start' return records From e768fac3876502595e7e6c68ca06eb0304cc2e8a Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 14:17:12 -0700 Subject: [PATCH 09/34] rename parser library file --- lib/python/qmk/{compile_commands_json.py => build_log_parse.py} | 0 lib/python/qmk/cli/compiledb.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/python/qmk/{compile_commands_json.py => build_log_parse.py} (100%) diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/build_log_parse.py similarity index 100% rename from lib/python/qmk/compile_commands_json.py rename to lib/python/qmk/build_log_parse.py diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 0878161eae22..c9ec4a2c56e1 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -7,7 +7,7 @@ from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import create_make_command -from qmk.compile_commands_json import parse_make_n +from qmk.build_log_parse import parse_make_n from qmk.constants import QMK_FIRMWARE import json From 21c5b6302ee5b256dca41649d7d360dd8cc8299d Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 14:23:04 -0700 Subject: [PATCH 10/34] move lib functions into command file. there are only 2 and they aren't large --- lib/python/qmk/build_log_parse.py | 66 ------------------------------- lib/python/qmk/cli/compiledb.py | 63 ++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 71 deletions(-) delete mode 100755 lib/python/qmk/build_log_parse.py diff --git a/lib/python/qmk/build_log_parse.py b/lib/python/qmk/build_log_parse.py deleted file mode 100755 index 2d54dc180c33..000000000000 --- a/lib/python/qmk/build_log_parse.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -import sys -import re -from pathlib import Path -import json -import shlex -from functools import lru_cache -from subprocess import check_output - -from qmk.constants import QMK_FIRMWARE - -from typing import TextIO, List, Dict - - -@lru_cache(maxsize=10) -def system_libs(binary: str): - """Find the system include directory that the given build tool uses. - - Only tested on OSX+homebrew so far. - """ - - try: - return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include")) - except Exception: - return [] - - -file_re = re.compile(r"""printf "Compiling: ([^"]+)""") -cmd_re = re.compile(r"""LOG=\$\((.+?)\&\&""") - - -def parse_make_n(f: TextIO) -> List[Dict[str, str]]: - """parse the output of `make -n ` - - This function makes many assumptions about the format of your build log. - This happens to work right now for qmk. - """ - - state = 'start' - this_file = None - records = [] - for line in f: - if state == 'start': - m = file_re.search(line) - if m: - this_file = m.group(1) - state = 'cmd' - - if state == 'cmd': - m = cmd_re.search(line) - if m: - # we have a hit! - this_cmd = m.group(1) - args = shlex.split(this_cmd) - args += ['-I%s' % s for s in system_libs(args[0])] - new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') - records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) - state = 'start' - - return records - - -if __name__ == '__main__': - with open(sys.argv[1]) as f: - print(json.dumps(parse_make_n(f), indent=4)) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index c9ec4a2c56e1..a23b6f0fc799 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -1,23 +1,76 @@ """Creates a compilation database for the given keyboard build. """ + +import json +import re +import shlex import subprocess +import sys from argparse import FileType +from functools import lru_cache +from pathlib import Path +from subprocess import check_output +from typing import Dict, List, TextIO from milc import cli - -from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import create_make_command -from qmk.build_log_parse import parse_make_n from qmk.constants import QMK_FIRMWARE -import json +from qmk.decorators import automagic_keyboard, automagic_keymap + + +@lru_cache(maxsize=10) +def system_libs(binary: str): + """Find the system include directory that the given build tool uses. + + Only tested on OSX+homebrew so far. + """ + + try: + return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include")) + except Exception: + return [] + + +file_re = re.compile(r"""printf "Compiling: ([^"]+)""") +cmd_re = re.compile(r"""LOG=\$\((.+?)\&\&""") + + +def parse_make_n(f: TextIO) -> List[Dict[str, str]]: + """parse the output of `make -n ` + + This function makes many assumptions about the format of your build log. + This happens to work right now for qmk. + """ + + state = 'start' + this_file = None + records = [] + for line in f: + if state == 'start': + m = file_re.search(line) + if m: + this_file = m.group(1) + state = 'cmd' + + if state == 'cmd': + m = cmd_re.search(line) + if m: + # we have a hit! + this_cmd = m.group(1) + args = shlex.split(this_cmd) + args += ['-I%s' % s for s in system_libs(args[0])] + new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') + records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) + state = 'start' + return records @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.subcommand('Create a compilation database.') @automagic_keyboard @automagic_keymap -def compile(cli): +def compiledb(cli): """Creates a compilation database for the given keyboard build. Does a make clean, then a make -n for this target and uses the dry-run output to create From 3fe78179b6088fd8e908b11a1368a17dcd61cc12 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 14:36:33 -0700 Subject: [PATCH 11/34] currently debugging... --- lib/python/qmk/cli/__init__.py | 1 + lib/python/qmk/cli/compiledb.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 394a1353bc4e..b46454a48772 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -8,6 +8,7 @@ from . import cformat from . import compile +from . import compiledb from . import config from . import docs from . import doctor diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index a23b6f0fc799..2fe6de14dd35 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -65,6 +65,7 @@ def parse_make_n(f: TextIO) -> List[Dict[str, str]]: return records + @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.subcommand('Create a compilation database.') @@ -94,7 +95,7 @@ def compiledb(cli): cli.log.info('Making clean') subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - cli.log.info('Gathering build instructions') + cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) db = parse_make_n(proc.stdout) res = proc.wait() From ec3739b2e2cfdb8cb67d9a81a4c9311f93fe93aa Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 15:11:05 -0700 Subject: [PATCH 12/34] more robustly find config --- lib/python/qmk/cli/compiledb.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 2fe6de14dd35..252806e8782c 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -13,6 +13,8 @@ from typing import Dict, List, TextIO from milc import cli + +import qmk.path from qmk.commands import create_make_command from qmk.constants import QMK_FIRMWARE from qmk.decorators import automagic_keyboard, automagic_keymap @@ -81,14 +83,18 @@ def compiledb(cli): https://clang.llvm.org/docs/JSONCompilationDatabase.html """ command = None + # check both config domains: the magic decorator fills in `compiledb` but the user is + # more likely to have set `compile` in their config file. + current_keyboard = cli.config.compiledb.keyboard or cli.config.compile.keyboard + current_keymap = cli.config.compiledb.keymap or cli.config.compile.keymap - if cli.config.compile.keyboard and cli.config.compile.keymap: + if current_keyboard and current_keymap: # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=True) + command = create_make_command(current_keyboard, current_keymap, dry_run=True) - elif not cli.config.compile.keyboard: + elif not current_keyboard: cli.log.error('Could not determine keyboard!') - elif not cli.config.compile.keymap: + elif not current_keymap: cli.log.error('Could not determine keymap!') if command: From bf2aef254ebe326574194743820c3cfdb990c8d5 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 15:20:53 -0700 Subject: [PATCH 13/34] updated docs --- docs/cli_commands.md | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 52d59a13517a..decf3d669d78 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -71,23 +71,6 @@ $ qmk compile -kb dz60 ... ``` -**Creating a `compile_commands.json`** - -Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you. - -**Example:** - -``` -$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak -$ qmk compile -c -Ψ Making clean -Ψ Gathering build instructions -Ψ Found 50 compile commands -Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json -``` - -Now open your dev environment and live a squiggly-free life. - ## `qmk flash` This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default. @@ -221,6 +204,27 @@ qmk cformat qmk cformat -b branch_name ``` +## `qmk compiledb` + +Creates a `compile_commands.json` file. + +Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you. + +This command needs to know which keyboard and keymap to build. It uses the same configuration options as the `qmk compile` command: arguments, current directory, and config files. + +**Example:** + +``` +$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak +$ qmk compiledb +Ψ Making clean +Ψ Gathering build instructions from make -n gh60/satan:colemak +Ψ Found 50 compile commands +Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json +``` + +Now open your dev environment and live a squiggly-free life. + ## `qmk docs` This command starts a local HTTP server which you can use for browsing or improving the docs. Default port is 8936. From 10171f8b83146c963d2b744bcc5ce7cc3f932f92 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 15:23:40 -0700 Subject: [PATCH 14/34] remove unused imports --- lib/python/qmk/cli/compiledb.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 252806e8782c..54e1dc5ce99c 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -5,8 +5,6 @@ import re import shlex import subprocess -import sys -from argparse import FileType from functools import lru_cache from pathlib import Path from subprocess import check_output @@ -14,7 +12,6 @@ from milc import cli -import qmk.path from qmk.commands import create_make_command from qmk.constants import QMK_FIRMWARE from qmk.decorators import automagic_keyboard, automagic_keymap From 697ca67707c4742fe136abd23b8e5c16d77d0919 Mon Sep 17 00:00:00 2001 From: Xton Date: Mon, 18 May 2020 21:01:31 -0700 Subject: [PATCH 15/34] reuse make executable from the main make command --- lib/python/qmk/cli/compiledb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 54e1dc5ce99c..b851f76a9485 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -95,8 +95,10 @@ def compiledb(cli): cli.log.error('Could not determine keymap!') if command: - cli.log.info('Making clean') - subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + # re-use same executable as the main make invocation (might be gmake) + clean_command = [command[0], 'clean'] + cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) + subprocess.run(clean_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) From 82a8be33f2451bf8241157d99aebe2d367a40b28 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 13:40:16 -0700 Subject: [PATCH 16/34] pulled source from dev branch --- .gitignore | 4 ++ lib/python/qmk/cli/compile.py | 36 +++++++++++++--- lib/python/qmk/compile_commands_json.py | 56 +++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100755 lib/python/qmk/compile_commands_json.py diff --git a/.gitignore b/.gitignore index 91d283e69ba7..c3a90db25d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,7 @@ __pycache__ # Allow to exist but don't include it in the repo user_song_list.h + +# clangd +compile_commands.json +.clangd/ diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 341f365f8c17..7ed099d22e54 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -9,12 +9,16 @@ from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json +from qmk.compile_commands_json import parse_make_n, qmk_dir +from pathlib import Path +import json @cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export to compile') @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") +@cli.argument('-b', '--builddb', arg_only=True, action='store_true', help="Does a make clean, then a make -n, and uses the dry-run output to create a compilation database (compile_commands.json). This file can help some IDEs and IDE-like editors work better. For more information about this: https://clang.llvm.org/docs/JSONCompilationDatabase.html") @cli.subcommand('Compile a QMK Firmware.') @automagic_keyboard @automagic_keymap @@ -36,19 +40,39 @@ def compile(cli): else: if cli.config.compile.keyboard and cli.config.compile.keymap: # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap) + dry_run = cli.args.builddb + command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=dry_run) elif not cli.config.compile.keyboard: cli.log.error('Could not determine keyboard!') elif not cli.config.compile.keymap: cli.log.error('Could not determine keymap!') - # Compile the firmware, if we're able to if command: - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - subprocess.run(command) + if cli.args.builddb: + cli.log.info('Making clean') + subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + cli.log.info('Gathering build instructions') + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + db = parse_make_n(proc.stdout) + res = proc.wait() + if res != 0: + raise RuntimeError(f"Got error from: {repr(command)}") + + cli.log.info(f"Found {len(db)} compile commands") + + dbpath = qmk_dir / 'compile_commands.json' + + cli.log.info(f"Writing build database to {dbpath}") + dbpath.write_text(json.dumps(db, indent=4)) + + else: + # Compile the firmware, if we're able to + cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) + if not cli.args.dry_run: + cli.echo('\n') + subprocess.run(command) else: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py new file mode 100755 index 000000000000..4aae872351bf --- /dev/null +++ b/lib/python/qmk/compile_commands_json.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +import sys +import re +import inspect +from pathlib import Path +import json +import shlex +from itertools import takewhile +from functools import lru_cache +from subprocess import check_output + + +from typing import TextIO, List, Dict + +qmk_dir = Path(__file__).parent.parent.parent.parent + +@lru_cache(maxsize=10) +def system_libs(binary: str): + try: + return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include")) + except Exception: + return [] + + +file_re = re.compile(r"""printf "Compiling: ([^"]+)""") +cmd_re = re.compile(r"""LOG=\$\(([^\)]+)\)""") + +def parse_make_n(f: TextIO) -> List[Dict[str,str]]: + state = 'start' + this_file = None + records = [] + for line in f: + if state == 'start': + m = file_re.search(line) + if m: + this_file = m.group(1) + state = 'cmd' + + if state == 'cmd': + m = cmd_re.search(line) + if m: + # we have a hit! + this_cmd = m.group(1) + args = shlex.split(this_cmd) + args = list(takewhile(lambda x: x != '&&', args)) + args += ['-I%s' % s for s in system_libs(args[0])] + new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') + records.append({"directory": str(qmk_dir), "command": new_cmd, "file": this_file}) + state = 'start' + + return records + +if __name__ == '__main__': + with open(sys.argv[1]) as f: + print(json.dumps(parse_make_n(f), indent=4)) From 3a951955b802086cd9c8fe333b8322151ca69ab9 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 13:52:20 -0700 Subject: [PATCH 17/34] missed a file from origin --- lib/python/qmk/commands.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 5a6e60988ade..b3be28bc8195 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -10,7 +10,7 @@ import qmk.keymap -def create_make_command(keyboard, keymap, target=None): +def create_make_command(keyboard, keymap, target=None, dry_run=False): """Create a make compile command Args: @@ -24,6 +24,9 @@ def create_make_command(keyboard, keymap, target=None): target Usually a bootloader. + dry_run + make -n -- don't actually build + Returns: A command that can be run to make the specified keyboard and keymap @@ -34,7 +37,10 @@ def create_make_command(keyboard, keymap, target=None): if target: make_args.append(target) - return [make_cmd, ':'.join(make_args)] + if dry_run: + return [make_cmd, '-n', ':'.join(make_args)] + else: + return [make_cmd, ':'.join(make_args)] def compile_configurator_json(user_keymap, bootloader=None): From 3ed55aa476dc5dc9628bbb4567692de8aa2f2d57 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 13:57:16 -0700 Subject: [PATCH 18/34] formatting --- lib/python/qmk/cli/compile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 7ed099d22e54..4e19bdf9e5ca 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -18,7 +18,11 @@ @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-b', '--builddb', arg_only=True, action='store_true', help="Does a make clean, then a make -n, and uses the dry-run output to create a compilation database (compile_commands.json). This file can help some IDEs and IDE-like editors work better. For more information about this: https://clang.llvm.org/docs/JSONCompilationDatabase.html") +@cli.argument('-b', '--builddb', arg_only=True, action='store_true', + help="Does a make clean, then a make -n for this target and uses the dry-run output to create " + "a compilation database (compile_commands.json). This file can help some IDEs and " + "IDE-like editors work better. For more information about this: " + "https://clang.llvm.org/docs/JSONCompilationDatabase.html") @cli.subcommand('Compile a QMK Firmware.') @automagic_keyboard @automagic_keymap From c9a4d97e1fea930d8fed37209266622a51ec1db3 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 14:18:02 -0700 Subject: [PATCH 19/34] revised argument names. relaxed matching rules to work for avr too --- lib/python/qmk/cli/compile.py | 6 +++--- lib/python/qmk/compile_commands_json.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 4e19bdf9e5ca..751563539051 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -18,7 +18,7 @@ @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-b', '--builddb', arg_only=True, action='store_true', +@cli.argument('-c', '--compile-commands', arg_only=True, action='store_true', help="Does a make clean, then a make -n for this target and uses the dry-run output to create " "a compilation database (compile_commands.json). This file can help some IDEs and " "IDE-like editors work better. For more information about this: " @@ -44,7 +44,7 @@ def compile(cli): else: if cli.config.compile.keyboard and cli.config.compile.keymap: # Generate the make command for a specific keyboard/keymap. - dry_run = cli.args.builddb + dry_run = cli.args.compile_commands command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=dry_run) elif not cli.config.compile.keyboard: @@ -53,7 +53,7 @@ def compile(cli): cli.log.error('Could not determine keymap!') if command: - if cli.args.builddb: + if cli.args.compile_commands: cli.log.info('Making clean') subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index 4aae872351bf..c6777cfcabaf 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -24,7 +24,7 @@ def system_libs(binary: str): file_re = re.compile(r"""printf "Compiling: ([^"]+)""") -cmd_re = re.compile(r"""LOG=\$\(([^\)]+)\)""") +cmd_re = re.compile(r"""LOG=\$\((.+)\&\&""") def parse_make_n(f: TextIO) -> List[Dict[str,str]]: state = 'start' @@ -43,7 +43,6 @@ def parse_make_n(f: TextIO) -> List[Dict[str,str]]: # we have a hit! this_cmd = m.group(1) args = shlex.split(this_cmd) - args = list(takewhile(lambda x: x != '&&', args)) args += ['-I%s' % s for s in system_libs(args[0])] new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') records.append({"directory": str(qmk_dir), "command": new_cmd, "file": this_file}) From 9226ba5aa6069bfd02fcd21f042adae948bef635 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 14:23:56 -0700 Subject: [PATCH 20/34] add docstrings --- lib/python/qmk/compile_commands_json.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index c6777cfcabaf..462be000af9f 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -15,8 +15,14 @@ qmk_dir = Path(__file__).parent.parent.parent.parent + @lru_cache(maxsize=10) def system_libs(binary: str): + """Find the system include directory that the given build tool uses. + + Only tested on OSX+homebrew so far. + """ + try: return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include")) except Exception: @@ -26,7 +32,14 @@ def system_libs(binary: str): file_re = re.compile(r"""printf "Compiling: ([^"]+)""") cmd_re = re.compile(r"""LOG=\$\((.+)\&\&""") -def parse_make_n(f: TextIO) -> List[Dict[str,str]]: + +def parse_make_n(f: TextIO) -> List[Dict[str, str]]: + """parse the output of `make -n ` + + This function makes many assumptions about the format of your build log. + This happens to work right now for qmk. + """ + state = 'start' this_file = None records = [] @@ -50,6 +63,7 @@ def parse_make_n(f: TextIO) -> List[Dict[str,str]]: return records + if __name__ == '__main__': with open(sys.argv[1]) as f: print(json.dumps(parse_make_n(f), indent=4)) From 764f35d26ff9b9f5f6694cc9c6107e7f79684616 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 14:41:33 -0700 Subject: [PATCH 21/34] added docs. tightened up regex --- docs/cli_commands.md | 17 +++++++++++++++++ lib/python/qmk/compile_commands_json.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index bb0de3c0db53..8a7d8526e344 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -73,6 +73,23 @@ $ qmk compile -kb dz60 ... ``` +**Creating a `compile_commands.json`** + +Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you. + +**Example:** + +``` +$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak +$ qmk compile -c +Ψ Making clean +Ψ Gathering build instructions +Ψ Found 50 compile commands +Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json +``` + +Now open your dev environment and live a squiggly-free life. + ## `qmk flash` This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default. To specify a different bootloader, use `-bl `. Visit the [Flashing Firmware](flashing.md) guide for more details of the available bootloaders. diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index 462be000af9f..d18979e6aab4 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -30,7 +30,7 @@ def system_libs(binary: str): file_re = re.compile(r"""printf "Compiling: ([^"]+)""") -cmd_re = re.compile(r"""LOG=\$\((.+)\&\&""") +cmd_re = re.compile(r"""LOG=\$\((.+?)\&\&""") def parse_make_n(f: TextIO) -> List[Dict[str, str]]: From b0a5e1a87e8cf29b8169799e6e91959456946a94 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 25 Apr 2020 14:52:51 -0700 Subject: [PATCH 22/34] remove unused imports --- lib/python/qmk/cli/compile.py | 1 - lib/python/qmk/compile_commands_json.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 751563539051..ec36c23ba0cb 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -10,7 +10,6 @@ from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json from qmk.compile_commands_json import parse_make_n, qmk_dir -from pathlib import Path import json diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index d18979e6aab4..e3ac188e96e7 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -2,11 +2,9 @@ import sys import re -import inspect from pathlib import Path import json import shlex -from itertools import takewhile from functools import lru_cache from subprocess import check_output From 67c08c613c2f02063645c6be9e582544681ef928 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 14:13:58 -0700 Subject: [PATCH 23/34] cleaning up command file. use existing qmk dir constant --- lib/python/qmk/cli/compile.py | 39 +++------------- lib/python/qmk/cli/compiledb.py | 61 +++++++++++++++++++++++++ lib/python/qmk/compile_commands_json.py | 5 +- 3 files changed, 69 insertions(+), 36 deletions(-) create mode 100755 lib/python/qmk/cli/compiledb.py diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index ec36c23ba0cb..341f365f8c17 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -9,19 +9,12 @@ from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json -from qmk.compile_commands_json import parse_make_n, qmk_dir -import json @cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export to compile') @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-c', '--compile-commands', arg_only=True, action='store_true', - help="Does a make clean, then a make -n for this target and uses the dry-run output to create " - "a compilation database (compile_commands.json). This file can help some IDEs and " - "IDE-like editors work better. For more information about this: " - "https://clang.llvm.org/docs/JSONCompilationDatabase.html") @cli.subcommand('Compile a QMK Firmware.') @automagic_keyboard @automagic_keymap @@ -43,39 +36,19 @@ def compile(cli): else: if cli.config.compile.keyboard and cli.config.compile.keymap: # Generate the make command for a specific keyboard/keymap. - dry_run = cli.args.compile_commands - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=dry_run) + command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap) elif not cli.config.compile.keyboard: cli.log.error('Could not determine keyboard!') elif not cli.config.compile.keymap: cli.log.error('Could not determine keymap!') + # Compile the firmware, if we're able to if command: - if cli.args.compile_commands: - cli.log.info('Making clean') - subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - cli.log.info('Gathering build instructions') - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - db = parse_make_n(proc.stdout) - res = proc.wait() - if res != 0: - raise RuntimeError(f"Got error from: {repr(command)}") - - cli.log.info(f"Found {len(db)} compile commands") - - dbpath = qmk_dir / 'compile_commands.json' - - cli.log.info(f"Writing build database to {dbpath}") - dbpath.write_text(json.dumps(db, indent=4)) - - else: - # Compile the firmware, if we're able to - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - subprocess.run(command) + cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) + if not cli.args.dry_run: + cli.echo('\n') + subprocess.run(command) else: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py new file mode 100755 index 000000000000..0878161eae22 --- /dev/null +++ b/lib/python/qmk/cli/compiledb.py @@ -0,0 +1,61 @@ +"""Creates a compilation database for the given keyboard build. +""" +import subprocess +from argparse import FileType + +from milc import cli + +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.commands import create_make_command +from qmk.compile_commands_json import parse_make_n +from qmk.constants import QMK_FIRMWARE +import json + + +@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.subcommand('Create a compilation database.') +@automagic_keyboard +@automagic_keymap +def compile(cli): + """Creates a compilation database for the given keyboard build. + + Does a make clean, then a make -n for this target and uses the dry-run output to create + a compilation database (compile_commands.json). This file can help some IDEs and + IDE-like editors work better. For more information about this: + + https://clang.llvm.org/docs/JSONCompilationDatabase.html + """ + command = None + + if cli.config.compile.keyboard and cli.config.compile.keymap: + # Generate the make command for a specific keyboard/keymap. + command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=True) + + elif not cli.config.compile.keyboard: + cli.log.error('Could not determine keyboard!') + elif not cli.config.compile.keymap: + cli.log.error('Could not determine keymap!') + + if command: + cli.log.info('Making clean') + subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + cli.log.info('Gathering build instructions') + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + db = parse_make_n(proc.stdout) + res = proc.wait() + if res != 0: + raise RuntimeError(f"Got error from: {repr(command)}") + + cli.log.info(f"Found {len(db)} compile commands") + + dbpath = QMK_FIRMWARE / 'compile_commands.json' + + cli.log.info(f"Writing build database to {dbpath}") + dbpath.write_text(json.dumps(db, indent=4)) + + else: + cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') + cli.echo('usage: qmk compiledb [-kb KEYBOARD] [-km KEYMAP]') + return False diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/compile_commands_json.py index e3ac188e96e7..2d54dc180c33 100755 --- a/lib/python/qmk/compile_commands_json.py +++ b/lib/python/qmk/compile_commands_json.py @@ -8,11 +8,10 @@ from functools import lru_cache from subprocess import check_output +from qmk.constants import QMK_FIRMWARE from typing import TextIO, List, Dict -qmk_dir = Path(__file__).parent.parent.parent.parent - @lru_cache(maxsize=10) def system_libs(binary: str): @@ -56,7 +55,7 @@ def parse_make_n(f: TextIO) -> List[Dict[str, str]]: args = shlex.split(this_cmd) args += ['-I%s' % s for s in system_libs(args[0])] new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') - records.append({"directory": str(qmk_dir), "command": new_cmd, "file": this_file}) + records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) state = 'start' return records From c0fc66ef38f742b36eefdce938ca5ae7d8f7d481 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 14:17:12 -0700 Subject: [PATCH 24/34] rename parser library file --- lib/python/qmk/{compile_commands_json.py => build_log_parse.py} | 0 lib/python/qmk/cli/compiledb.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/python/qmk/{compile_commands_json.py => build_log_parse.py} (100%) diff --git a/lib/python/qmk/compile_commands_json.py b/lib/python/qmk/build_log_parse.py similarity index 100% rename from lib/python/qmk/compile_commands_json.py rename to lib/python/qmk/build_log_parse.py diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 0878161eae22..c9ec4a2c56e1 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -7,7 +7,7 @@ from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import create_make_command -from qmk.compile_commands_json import parse_make_n +from qmk.build_log_parse import parse_make_n from qmk.constants import QMK_FIRMWARE import json From 2f103f506f25bf1ccd881a5f3b844b34bbe9688b Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 14:23:04 -0700 Subject: [PATCH 25/34] move lib functions into command file. there are only 2 and they aren't large --- lib/python/qmk/build_log_parse.py | 66 ------------------------------- lib/python/qmk/cli/compiledb.py | 63 ++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 71 deletions(-) delete mode 100755 lib/python/qmk/build_log_parse.py diff --git a/lib/python/qmk/build_log_parse.py b/lib/python/qmk/build_log_parse.py deleted file mode 100755 index 2d54dc180c33..000000000000 --- a/lib/python/qmk/build_log_parse.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -import sys -import re -from pathlib import Path -import json -import shlex -from functools import lru_cache -from subprocess import check_output - -from qmk.constants import QMK_FIRMWARE - -from typing import TextIO, List, Dict - - -@lru_cache(maxsize=10) -def system_libs(binary: str): - """Find the system include directory that the given build tool uses. - - Only tested on OSX+homebrew so far. - """ - - try: - return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include")) - except Exception: - return [] - - -file_re = re.compile(r"""printf "Compiling: ([^"]+)""") -cmd_re = re.compile(r"""LOG=\$\((.+?)\&\&""") - - -def parse_make_n(f: TextIO) -> List[Dict[str, str]]: - """parse the output of `make -n ` - - This function makes many assumptions about the format of your build log. - This happens to work right now for qmk. - """ - - state = 'start' - this_file = None - records = [] - for line in f: - if state == 'start': - m = file_re.search(line) - if m: - this_file = m.group(1) - state = 'cmd' - - if state == 'cmd': - m = cmd_re.search(line) - if m: - # we have a hit! - this_cmd = m.group(1) - args = shlex.split(this_cmd) - args += ['-I%s' % s for s in system_libs(args[0])] - new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') - records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) - state = 'start' - - return records - - -if __name__ == '__main__': - with open(sys.argv[1]) as f: - print(json.dumps(parse_make_n(f), indent=4)) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index c9ec4a2c56e1..a23b6f0fc799 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -1,23 +1,76 @@ """Creates a compilation database for the given keyboard build. """ + +import json +import re +import shlex import subprocess +import sys from argparse import FileType +from functools import lru_cache +from pathlib import Path +from subprocess import check_output +from typing import Dict, List, TextIO from milc import cli - -from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import create_make_command -from qmk.build_log_parse import parse_make_n from qmk.constants import QMK_FIRMWARE -import json +from qmk.decorators import automagic_keyboard, automagic_keymap + + +@lru_cache(maxsize=10) +def system_libs(binary: str): + """Find the system include directory that the given build tool uses. + + Only tested on OSX+homebrew so far. + """ + + try: + return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include")) + except Exception: + return [] + + +file_re = re.compile(r"""printf "Compiling: ([^"]+)""") +cmd_re = re.compile(r"""LOG=\$\((.+?)\&\&""") + + +def parse_make_n(f: TextIO) -> List[Dict[str, str]]: + """parse the output of `make -n ` + + This function makes many assumptions about the format of your build log. + This happens to work right now for qmk. + """ + + state = 'start' + this_file = None + records = [] + for line in f: + if state == 'start': + m = file_re.search(line) + if m: + this_file = m.group(1) + state = 'cmd' + + if state == 'cmd': + m = cmd_re.search(line) + if m: + # we have a hit! + this_cmd = m.group(1) + args = shlex.split(this_cmd) + args += ['-I%s' % s for s in system_libs(args[0])] + new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') + records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) + state = 'start' + return records @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.subcommand('Create a compilation database.') @automagic_keyboard @automagic_keymap -def compile(cli): +def compiledb(cli): """Creates a compilation database for the given keyboard build. Does a make clean, then a make -n for this target and uses the dry-run output to create From c03da56592c9469a8be08e662d11e96d6b836597 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 14:36:33 -0700 Subject: [PATCH 26/34] currently debugging... --- lib/python/qmk/cli/__init__.py | 1 + lib/python/qmk/cli/compiledb.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 47f60c601b12..31fc2837e7d3 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -8,6 +8,7 @@ from . import cformat from . import compile +from . import compiledb from . import config from . import docs from . import doctor diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index a23b6f0fc799..2fe6de14dd35 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -65,6 +65,7 @@ def parse_make_n(f: TextIO) -> List[Dict[str, str]]: return records + @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.subcommand('Create a compilation database.') @@ -94,7 +95,7 @@ def compiledb(cli): cli.log.info('Making clean') subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - cli.log.info('Gathering build instructions') + cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) db = parse_make_n(proc.stdout) res = proc.wait() From 8db4e134198b829f24980c507bbedbf3dfe3fd27 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 15:11:05 -0700 Subject: [PATCH 27/34] more robustly find config --- lib/python/qmk/cli/compiledb.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 2fe6de14dd35..252806e8782c 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -13,6 +13,8 @@ from typing import Dict, List, TextIO from milc import cli + +import qmk.path from qmk.commands import create_make_command from qmk.constants import QMK_FIRMWARE from qmk.decorators import automagic_keyboard, automagic_keymap @@ -81,14 +83,18 @@ def compiledb(cli): https://clang.llvm.org/docs/JSONCompilationDatabase.html """ command = None + # check both config domains: the magic decorator fills in `compiledb` but the user is + # more likely to have set `compile` in their config file. + current_keyboard = cli.config.compiledb.keyboard or cli.config.compile.keyboard + current_keymap = cli.config.compiledb.keymap or cli.config.compile.keymap - if cli.config.compile.keyboard and cli.config.compile.keymap: + if current_keyboard and current_keymap: # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, dry_run=True) + command = create_make_command(current_keyboard, current_keymap, dry_run=True) - elif not cli.config.compile.keyboard: + elif not current_keyboard: cli.log.error('Could not determine keyboard!') - elif not cli.config.compile.keymap: + elif not current_keymap: cli.log.error('Could not determine keymap!') if command: From 67ae7d8bee121bf447019a3497952ced372cf65c Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 15:20:53 -0700 Subject: [PATCH 28/34] updated docs --- docs/cli_commands.md | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 8a7d8526e344..f371a92edecd 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -73,23 +73,6 @@ $ qmk compile -kb dz60 ... ``` -**Creating a `compile_commands.json`** - -Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you. - -**Example:** - -``` -$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak -$ qmk compile -c -Ψ Making clean -Ψ Gathering build instructions -Ψ Found 50 compile commands -Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json -``` - -Now open your dev environment and live a squiggly-free life. - ## `qmk flash` This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default. To specify a different bootloader, use `-bl `. Visit the [Flashing Firmware](flashing.md) guide for more details of the available bootloaders. @@ -254,6 +237,27 @@ qmk cformat qmk cformat -b branch_name ``` +## `qmk compiledb` + +Creates a `compile_commands.json` file. + +Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you. + +This command needs to know which keyboard and keymap to build. It uses the same configuration options as the `qmk compile` command: arguments, current directory, and config files. + +**Example:** + +``` +$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak +$ qmk compiledb +Ψ Making clean +Ψ Gathering build instructions from make -n gh60/satan:colemak +Ψ Found 50 compile commands +Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json +``` + +Now open your dev environment and live a squiggly-free life. + ## `qmk docs` This command starts a local HTTP server which you can use for browsing or improving the docs. Default port is 8936. From 976b0b14cfc588ad21fd638cfa68174415d6f9a5 Mon Sep 17 00:00:00 2001 From: Xton Date: Sat, 9 May 2020 15:23:40 -0700 Subject: [PATCH 29/34] remove unused imports --- lib/python/qmk/cli/compiledb.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 252806e8782c..54e1dc5ce99c 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -5,8 +5,6 @@ import re import shlex import subprocess -import sys -from argparse import FileType from functools import lru_cache from pathlib import Path from subprocess import check_output @@ -14,7 +12,6 @@ from milc import cli -import qmk.path from qmk.commands import create_make_command from qmk.constants import QMK_FIRMWARE from qmk.decorators import automagic_keyboard, automagic_keymap From 7ab34654aa69ef65a446738f5796eeb9bc1faa49 Mon Sep 17 00:00:00 2001 From: Xton Date: Mon, 18 May 2020 21:01:31 -0700 Subject: [PATCH 30/34] reuse make executable from the main make command --- lib/python/qmk/cli/compiledb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 54e1dc5ce99c..b851f76a9485 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -95,8 +95,10 @@ def compiledb(cli): cli.log.error('Could not determine keymap!') if command: - cli.log.info('Making clean') - subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + # re-use same executable as the main make invocation (might be gmake) + clean_command = [command[0], 'clean'] + cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) + subprocess.run(clean_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) From c558c5da733600f9345cd5888546a1ad30e0cfc3 Mon Sep 17 00:00:00 2001 From: Xton Date: Mon, 7 Sep 2020 15:56:15 -0700 Subject: [PATCH 31/34] remove MAKEFLAGS from environment for better control over process management --- lib/python/qmk/cli/compiledb.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index b851f76a9485..1d3a61464043 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -5,6 +5,7 @@ import re import shlex import subprocess +import os from functools import lru_cache from pathlib import Path from subprocess import check_output @@ -95,13 +96,17 @@ def compiledb(cli): cli.log.error('Could not determine keymap!') if command: + # remove any environment variable overrides which could trip us up + env = os.environ.copy() + env.pop("MAKEFLAGS", None) + # re-use same executable as the main make invocation (might be gmake) clean_command = [command[0], 'clean'] cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) - subprocess.run(clean_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run(clean_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env) cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) db = parse_make_n(proc.stdout) res = proc.wait() if res != 0: From afb2527081135a8b92a14175ccbee69159eff456 Mon Sep 17 00:00:00 2001 From: Christon DeWan Date: Sat, 26 Sep 2020 22:49:33 -0700 Subject: [PATCH 32/34] Update .gitignore Co-authored-by: Michael Forster --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c3a90db25d6e..4546a4e7ee07 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,4 @@ user_song_list.h # clangd compile_commands.json .clangd/ +.cache/ From f64f64a88acbd7d10bae42cce9cc86c558f4e898 Mon Sep 17 00:00:00 2001 From: Xton Date: Sun, 11 Oct 2020 10:59:47 -0700 Subject: [PATCH 33/34] add a usage line to docs --- docs/cli_commands.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index f371a92edecd..5020b2be0574 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -239,6 +239,12 @@ qmk cformat -b branch_name ## `qmk compiledb` +**Usage**: + +``` +qmk compiledb [-kb KEYBOARD] [-km KEYMAP] +``` + Creates a `compile_commands.json` file. Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you. From 6cbcc9c105ffad0fabb34f49ea5bf949bbb4b6c8 Mon Sep 17 00:00:00 2001 From: Christon DeWan Date: Sun, 29 Nov 2020 22:37:08 -0800 Subject: [PATCH 34/34] doc change as suggested Co-authored-by: Nick Brassel --- lib/python/qmk/cli/compiledb.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/python/qmk/cli/compiledb.py b/lib/python/qmk/cli/compiledb.py index 1d3a61464043..19252fd8cab6 100755 --- a/lib/python/qmk/cli/compiledb.py +++ b/lib/python/qmk/cli/compiledb.py @@ -21,8 +21,6 @@ @lru_cache(maxsize=10) def system_libs(binary: str): """Find the system include directory that the given build tool uses. - - Only tested on OSX+homebrew so far. """ try: