2020 November 28 Breaking Changes Update (#11053)

* Branch point for 2020 November 28 Breaking Change                                                

* Remove matrix_col_t to allow MATRIX_ROWS > 32 (#10183)                                           

* Add support for soft serial to ATmega32U2 (#10204)                                               

* Change MIDI velocity implementation to allow direct control of velocity value (#9940)            

* Add ability to build a subset of all keyboards based on platform.                                

* Actually use eeprom_driver_init().                                                               

* Make bootloader_jump weak for ChibiOS. (#10417)                                                  

* Joystick 16-bit support (#10439)                                                                 

* Per-encoder resolutions (#10259)                                                                 

* Share button state from mousekey to pointing_device (#10179)                                     

* Add hotfix for chibios keyboards not wake (#10088)                                               

* Add advanced/efficient RGB Matrix Indicators (#8564)                                             

* Naming change.                                                                                   

* Support for STM32 GPIOF,G,H,I,J,K (#10206)                                                       

* Add milc as a dependency and remove the installed milc (#10563)                                  

* ChibiOS upgrade: early init conversions (#10214)                                                 

* ChibiOS upgrade: configuration file migrator (#9952)                                             

* Haptic and solenoid cleanup (#9700)                                                              

* XD75 cleanup (#10524)                                                                            

* OLED display update interval support (#10388)                                                    

* Add definition based on currently-selected serial driver. (#10716)                               

* New feature: Retro Tapping per key (#10622)                                                      

* Allow for modification of output RGB values when using rgblight/rgb_matrix. (#10638)             

* Add housekeeping task callbacks so that keyboards/keymaps are capable of executing code for each main loop iteration. (#10530)

* Rescale both ChibiOS and AVR backlighting.                                                       

* Reduce Helix keyboard build variation (#8669)                                                    

* Minor change to behavior allowing display updates to continue between task ticks (#10750)        

* Some GPIO manipulations in matrix.c change to atomic. (#10491)                                   

* qmk cformat (#10767)                                                                             

* [Keyboard] Update the Speedo firmware for v3.0 (#10657)                                          

* Maartenwut/Maarten namechange to evyd13/Evy (#10274)                                             

* [quantum] combine repeated lines of code (#10837)                                                

* Add step sequencer feature (#9703)                                                               

* aeboards/ext65 refactor (#10820)                                                                 

* Refactor xelus/dawn60 for Rev2 later (#10584)                                                    

* add DEBUG_MATRIX_SCAN_RATE_ENABLE to common_features.mk (#10824)                                 

* [Core] Added `add_oneshot_mods` & `del_oneshot_mods` (#10549)                                    

* update chibios os usb for the otg driver (#8893)                                                 

* Remove HD44780 References, Part 4 (#10735)                                                       

* [Keyboard] Add Valor FRL TKL (+refactor) (#10512)                                                

* Fix cursor position bug in oled_write_raw functions (#10800)                                     

* Fixup version.h writing when using SKIP_VERSION=yes (#10972)                                     

* Allow for certain code in the codebase assuming length of string. (#10974)                       

* Add AT90USB support for serial.c (#10706)                                                        

* Auto shift: support repeats and early registration (#9826)                                       

* Rename ledmatrix.h to match .c file (#7949)                                                      

* Split RGB_MATRIX_ENABLE into _ENABLE and _DRIVER (#10231)                                        

* Split LED_MATRIX_ENABLE into _ENABLE and _DRIVER (#10840)                                        

* Merge point for 2020 Nov 28 Breaking Change

lib/python/qmk/cli/__init__.py

@@ -8,6 +8,7 @@ from milc import cli
 
 from . import c2json
 from . import cformat
+from . import chibios
 from . import clean
 from . import compile
 from . import config

lib/python/qmk/cli/chibios/__init__.py

@@ -0,0 +1 @@
+from . import confmigrate

lib/python/qmk/cli/chibios/confmigrate.py

@@ -0,0 +1,161 @@
+"""This script automates the copying of the default keymap into your own keymap.
+"""
+import re
+import sys
+import os
+
+from qmk.constants import QMK_FIRMWARE
+from qmk.path import normpath
+from milc import cli
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+fileHeader = """\
+/* Copyright 2020 QMK
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * This file was auto-generated by:
+ *    `qmk chibios-confupdate -i {0} -r {1}`
+ */
+
+#pragma once
+"""
+
+
+def collect_defines(filepath):
+    with open(filepath, 'r') as f:
+        content = f.read()
+        define_search = re.compile(r'(?m)^#\s*define\s+(?:.*\\\r?\n)*.*$', re.MULTILINE)
+        value_search = re.compile(r'^#\s*define\s+(?P<name>[a-zA-Z0-9_]+(\([^\)]*\))?)\s*(?P<value>.*)', re.DOTALL)
+        define_matches = define_search.findall(content)
+
+        defines = {"keys": [], "dict": {}}
+        for define_match in define_matches:
+            value_match = value_search.search(define_match)
+            defines["keys"].append(value_match.group("name"))
+            defines["dict"][value_match.group("name")] = value_match.group("value")
+        return defines
+
+
+def check_diffs(input_defs, reference_defs):
+    not_present_in_input = []
+    not_present_in_reference = []
+    to_override = []
+
+    for key in reference_defs["keys"]:
+        if key not in input_defs["dict"]:
+            not_present_in_input.append(key)
+            continue
+
+    for key in input_defs["keys"]:
+        if key not in input_defs["dict"]:
+            not_present_in_input.append(key)
+            continue
+
+    for key in input_defs["keys"]:
+        if key in reference_defs["keys"] and input_defs["dict"][key] != reference_defs["dict"][key]:
+            to_override.append((key, input_defs["dict"][key]))
+
+    return (to_override, not_present_in_input, not_present_in_reference)
+
+
+def migrate_chconf_h(to_override, outfile):
+    print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
+
+    for override in to_override:
+        print("#define %s %s" % (override[0], override[1]), file=outfile)
+        print("", file=outfile)
+
+    print("#include_next <chconf.h>\n", file=outfile)
+
+
+def migrate_halconf_h(to_override, outfile):
+    print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
+
+    for override in to_override:
+        print("#define %s %s" % (override[0], override[1]), file=outfile)
+        print("", file=outfile)
+
+    print("#include_next <halconf.h>\n", file=outfile)
+
+
+def migrate_mcuconf_h(to_override, outfile):
+    print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
+
+    print("#include_next <mcuconf.h>\n", file=outfile)
+
+    for override in to_override:
+        print("#undef %s" % (override[0]), file=outfile)
+        print("#define %s %s" % (override[0], override[1]), file=outfile)
+        print("", file=outfile)
+
+
+@cli.argument('-i', '--input', type=normpath, arg_only=True, help='Specify input config file.')
+@cli.argument('-r', '--reference', type=normpath, arg_only=True, help='Specify the reference file to compare against')
+@cli.argument('-o', '--overwrite', arg_only=True, action='store_true', help='Overwrites the input file during migration.')
+@cli.argument('-d', '--delete', arg_only=True, action='store_true', help='If the file has no overrides, migration will delete the input file.')
+@cli.subcommand('Generates a migrated ChibiOS configuration file, as a result of comparing the input against a reference')
+def chibios_confmigrate(cli):
+    """Generates a usable ChibiOS replacement configuration file, based on a fully-defined conf and a reference config.
+    """
+
+    input_defs = collect_defines(cli.args.input)
+    reference_defs = collect_defines(cli.args.reference)
+
+    (to_override, not_present_in_input, not_present_in_reference) = check_diffs(input_defs, reference_defs)
+
+    if len(not_present_in_input) > 0:
+        eprint("Keys not in input, but present inside reference (potential manual migration required):")
+        for key in not_present_in_input:
+            eprint("   %s" % (key))
+
+    if len(not_present_in_reference) > 0:
+        eprint("Keys not in reference, but present inside input (potential manual migration required):")
+        for key in not_present_in_reference:
+            eprint("   %s" % (key))
+
+    if len(to_override) == 0:
+        eprint('No overrides found! If there were no missing keys above, it should be safe to delete the input file.')
+        if cli.args.delete:
+            os.remove(cli.args.input)
+    else:
+        eprint('Overrides found:')
+        for override in to_override:
+            eprint("%40s: %s -> %s" % (override[0], reference_defs["dict"][override[0]].encode('unicode_escape').decode("utf-8"), override[1].encode('unicode_escape').decode("utf-8")))
+
+        eprint('--------------------------------------')
+
+        if "CHCONF_H" in input_defs["dict"] or "_CHCONF_H_" in input_defs["dict"]:
+            migrate_chconf_h(to_override, outfile=sys.stdout)
+            if cli.args.overwrite:
+                with open(cli.args.input, "w") as out_file:
+                    migrate_chconf_h(to_override, outfile=out_file)
+
+        elif "HALCONF_H" in input_defs["dict"] or "_HALCONF_H_" in input_defs["dict"]:
+            migrate_halconf_h(to_override, outfile=sys.stdout)
+            if cli.args.overwrite:
+                with open(cli.args.input, "w") as out_file:
+                    migrate_halconf_h(to_override, outfile=out_file)
+
+        elif "MCUCONF_H" in input_defs["dict"] or "_MCUCONF_H_" in input_defs["dict"]:
+            migrate_mcuconf_h(to_override, outfile=sys.stdout)
+            if cli.args.overwrite:
+                with open(cli.args.input, "w") as out_file:
+                    migrate_mcuconf_h(to_override, outfile=out_file)

lib/python/qmk/cli/doctor.py

@@ -10,9 +10,9 @@ from pathlib import Path
 from enum import Enum
 
 from milc import cli
+from milc.questions import yesno
 from qmk import submodules
 from qmk.constants import QMK_FIRMWARE
-from qmk.questions import yesno
 from qmk.commands import run