[Tests] Increase QMK test coverage take 2 (#15269)

* Add per-test keymaps

* Add better trace and info logs for failed unit-tests

* Add layer state assertion with tracing message

* Use individual test binaries configuration options

* Add basic qmk functionality tests

* Add tap hold configurations tests

* Add auto shift tests

Co-authored-by: Nick Brassel <nick@tzarc.org>

tests/test_common/build.mk

@@ -0,0 +1,16 @@
+# Copyright 2021 Stefan Kerkmann
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+CUSTOM_MATRIX=yes

tests/test_common/keyboard_report_util.cpp

@@ -44,16 +44,21 @@ bool operator==(const report_keyboard_t& lhs, const report_keyboard_t& rhs) {
     return lhs.mods == rhs.mods && lhskeys == rhskeys;
 }
 
-std::ostream& operator<<(std::ostream& stream, const report_keyboard_t& value) {
-    stream << "Keyboard report:" << std::endl;
-    stream << "Mods: " << (uint32_t)value.mods << std::endl;
-    stream << "Keys: ";
+std::ostream& operator<<(std::ostream& stream, const report_keyboard_t& report) {
+    auto keys = get_keys(report);
+
     // TODO: This should probably print friendly names for the keys
-    for (uint32_t k : get_keys(value)) {
-        stream << k << " ";
+    stream << "Keyboard Report: Mods (" << (uint32_t)report.mods << ") Keys (";
+
+    for (auto key = keys.cbegin(); key != keys.cend();) {
+        stream << +(*key);
+        key++;
+        if (key != keys.cend()) {
+            stream << ",";
+        }
     }
-    stream << std::endl;
-    return stream;
+
+    return stream << ")" << std::endl;
 }
 
 KeyboardReportMatcher::KeyboardReportMatcher(const std::vector<uint8_t>& keys) {

tests/test_common/keymap.c

@@ -0,0 +1,33 @@
+/* Copyright 2017 Fred Sundvik
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "quantum.h"
+
+// clang-format off
+
+const uint16_t PROGMEM
+               keymaps[][MATRIX_ROWS][MATRIX_COLS] =
+        {
+            [0] =
+                {
+                   {KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO},
+                   {KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO},
+                   {KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO},
+                   {KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO},
+                },
+};
+
+// clang-format on

tests/test_common/test_common.h

@@ -0,0 +1,4 @@
+#pragma once
+
+#define MATRIX_ROWS 4
+#define MATRIX_COLS 10

tests/test_common/test_driver.cpp

@@ -27,7 +27,10 @@ TestDriver::~TestDriver() { m_this = nullptr; }
 
 uint8_t TestDriver::keyboard_leds(void) { return m_this->m_leds; }
 
-void TestDriver::send_keyboard(report_keyboard_t* report) { m_this->send_keyboard_mock(*report); }
+void TestDriver::send_keyboard(report_keyboard_t* report) {
+    test_logger.trace() << *report;
+    m_this->send_keyboard_mock(*report);
+}
 
 void TestDriver::send_mouse(report_mouse_t* report) { m_this->send_mouse_mock(*report); }
 

tests/test_common/test_driver.hpp

@@ -20,25 +20,26 @@
 #include <stdint.h>
 #include "host.h"
 #include "keyboard_report_util.hpp"
-
+#include "test_logger.hpp"
 
 class TestDriver {
-public:
+   public:
     TestDriver();
     ~TestDriver();
     void set_leds(uint8_t leds) { m_leds = leds; }
 
-    MOCK_METHOD1(send_keyboard_mock, void (report_keyboard_t&));
-    MOCK_METHOD1(send_mouse_mock, void (report_mouse_t&));
-    MOCK_METHOD1(send_system_mock, void (uint16_t));
-    MOCK_METHOD1(send_consumer_mock, void (uint16_t));
-private:
-    static uint8_t keyboard_leds(void);
-    static void send_keyboard(report_keyboard_t *report);
-    static void send_mouse(report_mouse_t* report);
-    static void send_system(uint16_t data);
-    static void send_consumer(uint16_t data);
-    host_driver_t m_driver;
-    uint8_t m_leds = 0;
+    MOCK_METHOD1(send_keyboard_mock, void(report_keyboard_t&));
+    MOCK_METHOD1(send_mouse_mock, void(report_mouse_t&));
+    MOCK_METHOD1(send_system_mock, void(uint16_t));
+    MOCK_METHOD1(send_consumer_mock, void(uint16_t));
+
+   private:
+    static uint8_t     keyboard_leds(void);
+    static void        send_keyboard(report_keyboard_t* report);
+    static void        send_mouse(report_mouse_t* report);
+    static void        send_system(uint16_t data);
+    static void        send_consumer(uint16_t data);
+    host_driver_t      m_driver;
+    uint8_t            m_leds = 0;
     static TestDriver* m_this;
 };

tests/test_common/test_fixture.cpp

@@ -1,26 +1,48 @@
 #include "test_fixture.hpp"
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include "gmock/gmock-cardinalities.h"
 #include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "keyboard_report_util.hpp"
+#include "keycode.h"
 #include "test_driver.hpp"
+#include "test_logger.hpp"
 #include "test_matrix.h"
-#include "keyboard.h"
-#include "action.h"
-#include "action_tapping.h"
+#include "test_keymap_key.hpp"
 
 extern "C" {
+#include "action.h"
+#include "action_tapping.h"
+#include "action_util.h"
+#include "action_layer.h"
 #include "debug.h"
 #include "eeconfig.h"
-#include "action_layer.h"
+#include "keyboard.h"
+#include "keymap.h"
 
 void set_time(uint32_t t);
 void advance_time(uint32_t ms);
 }
 
 using testing::_;
-using testing::AnyNumber;
-using testing::Between;
-using testing::Return;
+
+/* This is used for dynamic dispatching keymap_key_to_keycode calls to the current active test_fixture. */
+TestFixture* TestFixture::m_this = nullptr;
+
+/* Override weak QMK function to allow the usage of isolated per-test keymaps in unit-tests.
+ * The actual call is dynamicaly dispatched to the current active test fixture, which in turn has it's own keymap. */
+extern "C" uint16_t keymap_key_to_keycode(uint8_t layer, keypos_t position) {
+    uint16_t keycode;
+    TestFixture::m_this->get_keycode(layer, position, &keycode);
+    return keycode;
+}
 
 void TestFixture::SetUpTestCase() {
+    test_logger.info() << "TestFixture setup-up start." << std::endl;
+
     // The following is enough to bootstrap the values set in main
     eeconfig_init_quantum();
     eeconfig_update_debug(debug_config.raw);
@@ -28,23 +50,99 @@ void TestFixture::SetUpTestCase() {
     TestDriver driver;
     EXPECT_CALL(driver, send_keyboard_mock(_));
     keyboard_init();
+
+    test_logger.info() << "TestFixture setup-up end." << std::endl;
 }
 
 void TestFixture::TearDownTestCase() {}
 
-TestFixture::TestFixture() {}
+TestFixture::TestFixture() { m_this = this; }
 
 TestFixture::~TestFixture() {
+    test_logger.info() << "TestFixture clean-up start." << std::endl;
     TestDriver driver;
-    // Run for a while to make sure all keys are completely released
-    EXPECT_CALL(driver, send_keyboard_mock(_)).Times(AnyNumber());
-    layer_clear();
+
+    EXPECT_CALL(driver, send_keyboard_mock(KeyboardReport())).Times(2);
+
+    /* Reset keyboard state. */
     clear_all_keys();
-    idle_for(TAPPING_TERM + 10);
+
+    clear_keyboard();
+
+    clear_oneshot_mods();
+    clear_oneshot_locked_mods();
+    reset_oneshot_layer();
+
+    layer_clear();
+
+#if defined(SWAP_HANDS_ENABLE)
+    clear_oneshot_swaphands();
+#endif
+
+    idle_for(TAPPING_TERM * 10);
     testing::Mock::VerifyAndClearExpectations(&driver);
-    // Verify that the matrix really is cleared
-    EXPECT_CALL(driver, send_keyboard_mock(KeyboardReport())).Times(0);
-    idle_for(TAPPING_TERM + 10);
+
+    /* Verify that the matrix really is cleared */
+    EXPECT_CALL(driver, send_keyboard_mock(_)).Times(0);
+    idle_for(TAPPING_TERM * 10);
+    testing::Mock::VerifyAndClearExpectations(&driver);
+
+    m_this = nullptr;
+
+    test_logger.info() << "TestFixture clean-up end." << std::endl;
+
+    print_test_log();
+}
+
+void TestFixture::add_key(KeymapKey key) {
+    if (this->find_key(key.layer, key.position)) {
+        FAIL() << "Key is already mapped for layer " << +key.layer << " and (column,row) (" << +key.position.col << "," << +key.position.row << ")";
+    }
+
+    this->keymap.push_back(key);
+}
+
+void TestFixture::set_keymap(std::initializer_list<KeymapKey> keys) {
+    this->keymap.clear();
+    for (auto& key : keys) {
+        add_key(key);
+    }
+}
+
+const KeymapKey* TestFixture::find_key(layer_t layer, keypos_t position) const {
+    auto keymap_key_predicate = [&](KeymapKey candidate) { return candidate.layer == layer && candidate.position.col == position.col && candidate.position.row == position.row; };
+
+    auto result = std::find_if(this->keymap.begin(), this->keymap.end(), keymap_key_predicate);
+
+    if (result != std::end(this->keymap)) {
+        return &(*result);
+    }
+    return nullptr;
+}
+
+void TestFixture::get_keycode(const layer_t layer, const keypos_t position, uint16_t* result) const {
+    bool key_is_out_of_bounds = position.col >= MATRIX_COLS && position.row >= MATRIX_ROWS;
+
+    if (key_is_out_of_bounds) {
+        /* See if this is done in hardware as well, because this is 100% out of bounds reads on all QMK keebs out there. */
+        auto msg = [&]() {
+            std::stringstream msg;
+            msg << "Keycode for position (" << +position.col << "," << +position.row << ") requested! This is out of bounds." << std::endl;
+            return msg.str();
+        }();
+
+        *result = KC_NO;
+        test_logger.error() << msg;
+        EXPECT_FALSE(key_is_out_of_bounds) << msg;
+        return;
+    }
+
+    if (auto key = this->find_key(layer, position)) {
+        *result = key->code;
+        return;
+    }
+
+    FAIL() << "No key is mapped for layer " << +layer << " and (column,row) " << +position.col << "," << +position.row << ")";
 }
 
 void TestFixture::run_one_scan_loop() {
@@ -57,3 +155,17 @@ void TestFixture::idle_for(unsigned time) {
         run_one_scan_loop();
     }
 }
+
+void TestFixture::print_test_log() const {
+    const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
+    if (HasFailure()) {
+        std::cerr << test_info->test_case_name() << "." << test_info->name() << " failed!" << std::endl;
+        test_logger.print_log();
+    }
+    test_logger.reset();
+}
+
+void TestFixture::expect_layer_state(layer_t layer_state) const {
+    test_logger.trace() << "Layer state: (" << +layer_state << ") Highest layer bit: (" << +get_highest_layer(layer_state) << ")" << std::endl;
+    EXPECT_TRUE(layer_state_is(layer_state));
+}

tests/test_common/test_fixture.hpp

@@ -16,15 +16,34 @@
 
 #pragma once
 
+#include <cstdint>
+#include <unordered_map>
+#include <optional>
 #include "gtest/gtest.h"
+#include "keyboard.h"
+#include "test_keymap_key.hpp"
 
 class TestFixture : public testing::Test {
-public:
+   public:
+    static TestFixture* m_this;
+
     TestFixture();
     ~TestFixture();
     static void SetUpTestCase();
     static void TearDownTestCase();
 
+    void set_keymap(std::initializer_list<KeymapKey> keycodes);
+    void add_key(const KeymapKey key);
+
+    const KeymapKey* find_key(const layer_t layer_t, const keypos_t position) const;
+    void                     get_keycode(const layer_t layer, const keypos_t position, uint16_t* result) const;
+
     void run_one_scan_loop();
     void idle_for(unsigned ms);
+
+    void expect_layer_state(layer_t layer) const;
+
+   protected:
+    void                   print_test_log() const;
+    std::vector<KeymapKey> keymap;
 };

tests/test_common/test_keymap_key.cpp

@@ -0,0 +1,30 @@
+/* Copyright 2021 Stefan Kerkmann
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "test_keymap_key.hpp"
+#include "test_logger.hpp"
+#include "gtest/gtest-message.h"
+#include "gtest/gtest.h"
+
+void KeymapKey::press() {
+    test_logger.trace() << "Key pressed:  (" << +this->position.col << "," << +this->position.row << ")" << std::endl;
+    press_key(this->position.col, this->position.row);
+}
+
+void KeymapKey::release() {
+    test_logger.trace() << "Key released: (" << +this->position.col << "," << +this->position.row << ")" << std::endl;
+    release_key(this->position.col, this->position.row);
+}

tests/test_common/test_keymap_key.hpp

@@ -0,0 +1,46 @@
+/* Copyright 2021 Stefan Kerkmann
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+extern "C" {
+#include "keyboard.h"
+#include "test_matrix.h"
+}
+
+#include <cassert>
+
+typedef uint8_t layer_t;
+
+struct KeymapKey {
+    KeymapKey(layer_t layer, uint8_t col, uint8_t row, uint16_t keycode) : layer(layer), position({.col = col, .row = row}), code(keycode), report_code(keycode) { validate(); }
+    KeymapKey(layer_t layer, uint8_t col, uint8_t row, uint16_t keycode, uint16_t report_code) : layer(layer), position({.col = col, .row = row}), code(keycode), report_code(report_code) { validate(); }
+
+    void press();
+    void release();
+
+    const layer_t  layer;
+    const keypos_t position;
+    const uint16_t code;
+    /* Sometimes the keycode does not match the code that is send in the usb report, so we provide it here. */
+    const uint16_t report_code;
+
+   private:
+    void validate() {
+        assert(position.col <= MATRIX_COLS);
+        assert(position.row <= MATRIX_ROWS);
+    }
+};

tests/test_common/test_logger.cpp

@@ -0,0 +1,39 @@
+/* Copyright 2021 Stefan Kerkmann
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <iostream>
+#include "test_logger.hpp"
+
+TestLogger test_logger;
+
+TestLogger& TestLogger::info() {
+    *this << "[ INFO     ] ";
+    return *this;
+}
+
+TestLogger& TestLogger::trace() {
+    *this << "[ TRACE    ] ";
+    return *this;
+}
+
+TestLogger& TestLogger::error() {
+    *this << "[ ERROR    ] ";
+    return *this;
+}
+
+void TestLogger::reset() { this->m_log.str(""); };
+
+void TestLogger::print_log() { std::cerr << this->m_log.str(); }

tests/test_common/test_logger.hpp

@@ -0,0 +1,35 @@
+/* Copyright 2021 Stefan Kerkmann
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <ostream>
+#include <sstream>
+
+class TestLogger : public std::ostream {
+   public:
+    TestLogger() : std::ostream(&m_log){};
+    TestLogger& info();
+    TestLogger& trace();
+    TestLogger& error();
+    void print_log();
+    void reset();
+
+   private:
+    std::stringbuf m_log;
+};
+
+extern TestLogger test_logger;