CLI refactoring for common build target APIs (#22221)
This commit is contained in:
Nick Brassel
GitHub
parent
c4d3521ba6
commit
4938210711
10 changed files with 296 additions and 285 deletions
|
|
@ -2,7 +2,7 @@
|
|||
"""
|
||||
from subprocess import DEVNULL
|
||||
|
||||
from qmk.commands import create_make_target
|
||||
from qmk.commands import find_make
|
||||
from milc import cli
|
||||
|
||||
|
||||
|
|
@ -11,4 +11,4 @@ from milc import cli
|
|||
def clean(cli):
|
||||
"""Runs `make clean` (or `make distclean` if --all is passed)
|
||||
"""
|
||||
cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
|
||||
cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL)
|
||||
|
|
|
|||
|
|
@ -7,22 +7,11 @@ from argcomplete.completers import FilesCompleter
|
|||
from milc import cli
|
||||
|
||||
import qmk.path
|
||||
from qmk.constants import QMK_FIRMWARE
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap
|
||||
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
|
||||
from qmk.commands import build_environment
|
||||
from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
|
||||
from qmk.keymap import keymap_completer, locate_keymap
|
||||
from qmk.cli.generate.compilation_database import write_compilation_database
|
||||
|
||||
|
||||
def _is_keymap_target(keyboard, keymap):
|
||||
if keymap == 'all':
|
||||
return True
|
||||
|
||||
if locate_keymap(keyboard, keymap):
|
||||
return True
|
||||
|
||||
return False
|
||||
from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
|
||||
|
||||
|
||||
@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
|
||||
|
|
@ -32,6 +21,7 @@ def _is_keymap_target(keyboard, keymap):
|
|||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
|
||||
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
|
||||
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
|
||||
@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")
|
||||
@cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.")
|
||||
@cli.subcommand('Compile a QMK Firmware.')
|
||||
@automagic_keyboard
|
||||
|
|
@ -53,47 +43,27 @@ def compile(cli):
|
|||
# Build the environment vars
|
||||
envs = build_environment(cli.args.env)
|
||||
|
||||
# Determine the compile command
|
||||
commands = []
|
||||
|
||||
current_keyboard = None
|
||||
current_keymap = None
|
||||
# Handler for the build target
|
||||
target = None
|
||||
|
||||
if cli.args.filename:
|
||||
# If a configurator JSON was provided generate a keymap and compile it
|
||||
user_keymap = parse_configurator_json(cli.args.filename)
|
||||
commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
|
||||
# if we were given a filename, assume we have a json build target
|
||||
target = JsonKeymapBuildTarget(cli.args.filename)
|
||||
|
||||
elif cli.config.compile.keyboard and cli.config.compile.keymap:
|
||||
# Generate the make command for a specific keyboard/keymap.
|
||||
if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
|
||||
# if we got a keyboard and keymap, attempt to find it
|
||||
if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap):
|
||||
cli.log.error('Invalid keymap argument.')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
||||
if cli.args.clean:
|
||||
commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
|
||||
commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
|
||||
# If we got here, then we have a valid keyboard and keymap for a build target
|
||||
target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap)
|
||||
|
||||
current_keyboard = cli.config.compile.keyboard
|
||||
current_keymap = cli.config.compile.keymap
|
||||
|
||||
if not commands:
|
||||
if not target:
|
||||
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
||||
if cli.args.compiledb:
|
||||
if current_keyboard is None or current_keymap is None:
|
||||
cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.')
|
||||
cli.print_help()
|
||||
return False
|
||||
write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json')
|
||||
|
||||
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
|
||||
if not cli.args.dry_run:
|
||||
cli.echo('\n')
|
||||
for command in commands:
|
||||
ret = cli.run(command, capture_output=False)
|
||||
if ret.returncode:
|
||||
return ret.returncode
|
||||
target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
|
||||
target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs)
|
||||
|
|
|
|||
|
|
@ -19,13 +19,9 @@ from qmk.search import search_keymap_targets
|
|||
def find(cli):
|
||||
"""Search through all keyboards and keymaps for a given search criteria.
|
||||
"""
|
||||
targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter)
|
||||
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
|
||||
print(f'{target}')
|
||||
|
||||
if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
|
||||
cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
|
||||
|
||||
targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
|
||||
for keyboard, keymap, print_vals in targets:
|
||||
print(f'{keyboard}:{keymap}')
|
||||
|
||||
for key, val in print_vals:
|
||||
print(f' {key}={val}')
|
||||
for key in cli.args.print:
|
||||
print(f' {key}={target.dotty.get(key, None)}')
|
||||
|
|
|
|||
|
|
@ -4,25 +4,17 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
|
|||
A bootloader must be specified.
|
||||
"""
|
||||
from argcomplete.completers import FilesCompleter
|
||||
from pathlib import Path
|
||||
|
||||
from milc import cli
|
||||
|
||||
import qmk.path
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap
|
||||
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
|
||||
from qmk.commands import build_environment
|
||||
from qmk.keyboard import keyboard_completer, keyboard_folder
|
||||
from qmk.keymap import keymap_completer, locate_keymap
|
||||
from qmk.flashers import flasher
|
||||
|
||||
|
||||
def _is_keymap_target(keyboard, keymap):
|
||||
if keymap == 'all':
|
||||
return True
|
||||
|
||||
if locate_keymap(keyboard, keymap):
|
||||
return True
|
||||
|
||||
return False
|
||||
from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
|
||||
|
||||
|
||||
def _list_bootloaders():
|
||||
|
|
@ -89,7 +81,7 @@ def flash(cli):
|
|||
|
||||
If bootloader is omitted the make system will use the configured bootloader for that keyboard.
|
||||
"""
|
||||
if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
|
||||
if cli.args.filename and isinstance(cli.args.filename, Path) and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
|
||||
return _flash_binary(cli.args.filename, cli.args.mcu)
|
||||
|
||||
if cli.args.bootloaders:
|
||||
|
|
@ -98,34 +90,27 @@ def flash(cli):
|
|||
# Build the environment vars
|
||||
envs = build_environment(cli.args.env)
|
||||
|
||||
# Determine the compile command
|
||||
commands = []
|
||||
# Handler for the build target
|
||||
target = None
|
||||
|
||||
if cli.args.filename:
|
||||
# If a configurator JSON was provided generate a keymap and compile it
|
||||
user_keymap = parse_configurator_json(cli.args.filename)
|
||||
commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)]
|
||||
# if we were given a filename, assume we have a json build target
|
||||
target = JsonKeymapBuildTarget(cli.args.filename)
|
||||
|
||||
elif cli.config.flash.keyboard and cli.config.flash.keymap:
|
||||
# Generate the make command for a specific keyboard/keymap.
|
||||
if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
|
||||
# if we got a keyboard and keymap, attempt to find it
|
||||
if not locate_keymap(cli.config.flash.keyboard, cli.config.flash.keymap):
|
||||
cli.log.error('Invalid keymap argument.')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
||||
if cli.args.clean:
|
||||
commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
|
||||
commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))
|
||||
# If we got here, then we have a valid keyboard and keymap for a build target
|
||||
target = KeyboardKeymapBuildTarget(cli.config.flash.keyboard, cli.config.flash.keymap)
|
||||
|
||||
if not commands:
|
||||
if not target:
|
||||
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
||||
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
|
||||
if not cli.args.dry_run:
|
||||
cli.echo('\n')
|
||||
for command in commands:
|
||||
ret = cli.run(command, capture_output=False)
|
||||
if ret.returncode:
|
||||
return ret.returncode
|
||||
target.configure(parallel=cli.config.flash.parallel, clean=cli.args.clean)
|
||||
target.compile(cli.args.bootloader, dry_run=cli.args.dry_run, **envs)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from typing import Dict, Iterator, List, Union
|
|||
|
||||
from milc import cli, MILC
|
||||
|
||||
from qmk.commands import create_make_command
|
||||
from qmk.commands import find_make
|
||||
from qmk.constants import QMK_FIRMWARE
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap
|
||||
from qmk.keyboard import keyboard_completer, keyboard_folder
|
||||
|
|
@ -76,9 +76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
|
|||
return records
|
||||
|
||||
|
||||
def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> bool:
|
||||
def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool:
|
||||
# Generate the make command for a specific keyboard/keymap.
|
||||
command = create_make_command(keyboard, keymap, dry_run=True)
|
||||
if not command:
|
||||
from qmk.build_targets import KeyboardKeymapBuildTarget # Lazy load due to circular references
|
||||
target = KeyboardKeymapBuildTarget(keyboard, keymap)
|
||||
command = target.compile_command(dry_run=True, **env_vars)
|
||||
|
||||
if not command:
|
||||
cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
|
||||
|
|
@ -90,9 +93,10 @@ def write_compilation_database(keyboard: str, keymap: str, output_path: Path) ->
|
|||
env.pop("MAKEFLAGS", None)
|
||||
|
||||
# re-use same executable as the main make invocation (might be gmake)
|
||||
clean_command = [command[0], 'clean']
|
||||
cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
|
||||
cli.run(clean_command, capture_output=False, check=True, env=env)
|
||||
if not skip_clean:
|
||||
clean_command = [find_make(), "clean"]
|
||||
cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
|
||||
cli.run(clean_command, capture_output=False, check=True, env=env)
|
||||
|
||||
cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command))
|
||||
|
||||
|
|
|
|||
|
|
@ -3,26 +3,28 @@
|
|||
This will compile everything in parallel, for testing purposes.
|
||||
"""
|
||||
import os
|
||||
from typing import List
|
||||
from pathlib import Path
|
||||
from subprocess import DEVNULL
|
||||
from milc import cli
|
||||
|
||||
from qmk.constants import QMK_FIRMWARE
|
||||
from qmk.commands import _find_make, get_make_parallel_args
|
||||
from qmk.commands import find_make, get_make_parallel_args, build_environment
|
||||
from qmk.search import search_keymap_targets, search_make_targets
|
||||
from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget
|
||||
|
||||
|
||||
def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
|
||||
def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env):
|
||||
if len(targets) == 0:
|
||||
return
|
||||
|
||||
make_cmd = _find_make()
|
||||
make_cmd = find_make()
|
||||
builddir = Path(QMK_FIRMWARE) / '.build'
|
||||
makefile = builddir / 'parallel_kb_builds.mk'
|
||||
|
||||
if dry_run:
|
||||
cli.log.info('Compilation targets:')
|
||||
for target in sorted(targets):
|
||||
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
|
||||
cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}")
|
||||
else:
|
||||
if clean:
|
||||
|
|
@ -30,9 +32,13 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
|
|||
|
||||
builddir.mkdir(parents=True, exist_ok=True)
|
||||
with open(makefile, "w") as f:
|
||||
for target in sorted(targets):
|
||||
keyboard_name = target[0]
|
||||
keymap_name = target[1]
|
||||
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
|
||||
keyboard_name = target.keyboard
|
||||
keymap_name = target.keymap
|
||||
target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation
|
||||
target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
|
||||
command = target.compile_command(**env)
|
||||
command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism
|
||||
keyboard_safe = keyboard_name.replace('/', '_')
|
||||
build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
|
||||
failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
|
||||
|
|
@ -43,7 +49,7 @@ all: {keyboard_safe}_{keymap_name}_binary
|
|||
{keyboard_safe}_{keymap_name}_binary:
|
||||
@rm -f "{build_log}" || true
|
||||
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
|
||||
+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(env)} \\
|
||||
{' '.join(command)} \\
|
||||
>>"{build_log}" 2>&1 \\
|
||||
|| cp "{build_log}" "{failed_log}"
|
||||
@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
|
||||
|
|
@ -95,8 +101,11 @@ def mass_compile(cli):
|
|||
"""Compile QMK Firmware against all keyboards.
|
||||
"""
|
||||
if len(cli.args.builds) > 0:
|
||||
targets = search_make_targets(cli.args.builds, cli.args.filter)
|
||||
json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
|
||||
make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
|
||||
targets = search_make_targets(make_like_targets)
|
||||
targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])
|
||||
else:
|
||||
targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
|
||||
|
||||
return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env)
|
||||
return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue