Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

new qmk subcommand for creating compile_commands.json for better clangd/IDE interop #10264

Closed
wants to merge 37 commits into from
Closed
Changes from 34 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
47365ae
pulled source from dev branch
Apr 25, 2020
e0bff4d
missed a file from origin
Apr 25, 2020
5a51eaa
formatting
Apr 25, 2020
d817b7d
revised argument names. relaxed matching rules to work for avr too
Apr 25, 2020
30b564f
add docstrings
Apr 25, 2020
849a12f
added docs. tightened up regex
Apr 25, 2020
b671c40
remove unused imports
Apr 25, 2020
f42cadc
cleaning up command file. use existing qmk dir constant
May 9, 2020
e768fac
rename parser library file
May 9, 2020
21c5b63
move lib functions into command file. there are only 2 and they aren'…
May 9, 2020
3fe7817
currently debugging...
May 9, 2020
ec3739b
more robustly find config
May 9, 2020
bf2aef2
updated docs
May 9, 2020
10171f8
remove unused imports
May 9, 2020
697ca67
reuse make executable from the main make command
May 19, 2020
82a8be3
pulled source from dev branch
Apr 25, 2020
3a95195
missed a file from origin
Apr 25, 2020
3ed55aa
formatting
Apr 25, 2020
c9a4d97
revised argument names. relaxed matching rules to work for avr too
Apr 25, 2020
9226ba5
add docstrings
Apr 25, 2020
764f35d
added docs. tightened up regex
Apr 25, 2020
b0a5e1a
remove unused imports
Apr 25, 2020
67c08c6
cleaning up command file. use existing qmk dir constant
May 9, 2020
c0fc66e
rename parser library file
May 9, 2020
2f103f5
move lib functions into command file. there are only 2 and they aren'…
May 9, 2020
c03da56
currently debugging...
May 9, 2020
8db4e13
more robustly find config
May 9, 2020
67ae7d8
updated docs
May 9, 2020
976b0b1
remove unused imports
May 9, 2020
7ab3465
reuse make executable from the main make command
May 19, 2020
16c47fd
Merge branch 'compile_commands' of https://github.com/xton/qmk_firmwa…
Aug 24, 2020
70dfd86
Merge branch 'master' into compile_commands
Sep 7, 2020
c558c5d
remove MAKEFLAGS from environment for better control over process man…
Sep 7, 2020
afb2527
Update .gitignore
xton Sep 27, 2020
f64f64a
add a usage line to docs
Oct 11, 2020
3830823
Merge branch 'compile_commands' of https://github.com/xton/qmk_firmwa…
Oct 11, 2020
6cbcc9c
doc change as suggested
xton Nov 30, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -76,3 +76,8 @@ __pycache__

# Allow to exist but don't include it in the repo
user_song_list.h

# clangd
compile_commands.json
.clangd/
xton marked this conversation as resolved.
Show resolved Hide resolved
.cache/
21 changes: 21 additions & 0 deletions docs/cli_commands.md
Original file line number Diff line number Diff line change
@@ -237,6 +237,27 @@ qmk cformat
qmk cformat -b branch_name
```

## `qmk compiledb`

Creates a `compile_commands.json` file.

Does your IDE/editor use a language server but doesn't _quite_ find all the necessary include files? Do you hate red squigglies? Do you wish your editor could figure out `#include QMK_KEYBOARD_H`? You might need a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html)! The qmk tool can build this for you.

This command needs to know which keyboard and keymap to build. It uses the same configuration options as the `qmk compile` command: arguments, current directory, and config files.

**Example:**

tzarc marked this conversation as resolved.
Show resolved Hide resolved
```
$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak
$ qmk compiledb
Ψ Making clean
Ψ Gathering build instructions from make -n gh60/satan:colemak
Ψ Found 50 compile commands
Ψ Writing build database to /Users/you/src/qmk_firmware/compile_commands.json
```

Now open your dev environment and live a squiggly-free life.

## `qmk docs`

This command starts a local HTTP server which you can use for browsing or improving the docs. Default port is 8936.
1 change: 1 addition & 0 deletions lib/python/qmk/cli/__init__.py
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@

from . import cformat
from . import compile
from . import compiledb
from . import config
from . import docs
from . import doctor
125 changes: 125 additions & 0 deletions lib/python/qmk/cli/compiledb.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
"""Creates a compilation database for the given keyboard build.
"""

import json
import re
import shlex
import subprocess
import os
from functools import lru_cache
from pathlib import Path
from subprocess import check_output
from typing import Dict, List, TextIO

from milc import cli

from qmk.commands import create_make_command
from qmk.constants import QMK_FIRMWARE
from qmk.decorators import automagic_keyboard, automagic_keymap


@lru_cache(maxsize=10)
def system_libs(binary: str):
"""Find the system include directory that the given build tool uses.

Only tested on OSX+homebrew so far.
xton marked this conversation as resolved.
Show resolved Hide resolved
"""

try:
return list(Path(check_output(['which', binary]).rstrip().decode()).resolve().parent.parent.glob("*/include"))
except Exception:
return []


file_re = re.compile(r"""printf "Compiling: ([^"]+)""")
cmd_re = re.compile(r"""LOG=\$\((.+?)\&\&""")


def parse_make_n(f: TextIO) -> List[Dict[str, str]]:
"""parse the output of `make -n <target>`

This function makes many assumptions about the format of your build log.
This happens to work right now for qmk.
"""

state = 'start'
this_file = None
records = []
for line in f:
if state == 'start':
m = file_re.search(line)
if m:
this_file = m.group(1)
state = 'cmd'

if state == 'cmd':
m = cmd_re.search(line)
if m:
# we have a hit!
this_cmd = m.group(1)
args = shlex.split(this_cmd)
args += ['-I%s' % s for s in system_libs(args[0])]
new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork')
records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file})
state = 'start'

return records


@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
@cli.subcommand('Create a compilation database.')
@automagic_keyboard
@automagic_keymap
def compiledb(cli):
"""Creates a compilation database for the given keyboard build.

Does a make clean, then a make -n for this target and uses the dry-run output to create
a compilation database (compile_commands.json). This file can help some IDEs and
IDE-like editors work better. For more information about this:

https://clang.llvm.org/docs/JSONCompilationDatabase.html
"""
command = None
# check both config domains: the magic decorator fills in `compiledb` but the user is
# more likely to have set `compile` in their config file.
current_keyboard = cli.config.compiledb.keyboard or cli.config.compile.keyboard
current_keymap = cli.config.compiledb.keymap or cli.config.compile.keymap

if current_keyboard and current_keymap:
# Generate the make command for a specific keyboard/keymap.
command = create_make_command(current_keyboard, current_keymap, dry_run=True)

elif not current_keyboard:
cli.log.error('Could not determine keyboard!')
elif not current_keymap:
cli.log.error('Could not determine keymap!')

if command:
# remove any environment variable overrides which could trip us up
env = os.environ.copy()
env.pop("MAKEFLAGS", None)

# re-use same executable as the main make invocation (might be gmake)
clean_command = [command[0], 'clean']
cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
subprocess.run(clean_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env)

cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command))
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env)
db = parse_make_n(proc.stdout)
res = proc.wait()
if res != 0:
raise RuntimeError(f"Got error from: {repr(command)}")

cli.log.info(f"Found {len(db)} compile commands")

dbpath = QMK_FIRMWARE / 'compile_commands.json'

cli.log.info(f"Writing build database to {dbpath}")
dbpath.write_text(json.dumps(db, indent=4))

else:
cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
cli.echo('usage: qmk compiledb [-kb KEYBOARD] [-km KEYMAP]')
return False
10 changes: 8 additions & 2 deletions lib/python/qmk/commands.py
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
import qmk.keymap


def create_make_command(keyboard, keymap, target=None):
def create_make_command(keyboard, keymap, target=None, dry_run=False):
"""Create a make compile command

Args:
@@ -25,6 +25,9 @@ def create_make_command(keyboard, keymap, target=None):
target
Usually a bootloader.

dry_run
make -n -- don't actually build

Returns:

A command that can be run to make the specified keyboard and keymap
@@ -35,7 +38,10 @@ def create_make_command(keyboard, keymap, target=None):
if target:
make_args.append(target)

return [make_cmd, ':'.join(make_args)]
if dry_run:
return [make_cmd, '-n', ':'.join(make_args)]
else:
return [make_cmd, ':'.join(make_args)]


def compile_configurator_json(user_keymap, bootloader=None):