diff --git a/.gitmodules b/.gitmodules index 5dddd18..4c7cd9e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "examples/posix_fs/fs-test"] path = examples/posix_fs/fs-test url = https://github.com/sheinz/fs-test +[submodule "tests/unity"] + path = tests/unity + url = https://github.com/ThrowTheSwitch/Unity.git diff --git a/common.mk b/common.mk index c3923c8..4f26cb5 100644 --- a/common.mk +++ b/common.mk @@ -134,9 +134,15 @@ $$($(1)_OBJ_DIR)%.o: $$($(1)_REAL_ROOT)%.S $$($(1)_MAKEFILE) $(wildcard $(ROOT)* $$($(1)_CC_BASE) -c $$< -o $$@ $$($(1)_CC_BASE) -MM -MT $$@ -MF $$(@:.o=.d) $$< -# the component is shown to depend on both obj and source files so we get a meaningful error message -# for missing explicitly named source files -$$($(1)_AR): $$($(1)_OBJ_FILES) $$($(1)_SRC_FILES) +$(1)_AR_IN_FILES = $$($(1)_OBJ_FILES) + +# the component is shown to depend on both obj and source files so we get +# a meaningful error message for missing explicitly named source files +ifeq ($(INCLUDE_SRC_INTO_AR),1) + $(1)_SRC_IN_AR_FILES = $$($(1)_SRC_FILES) +endif + +$$($(1)_AR): $$($(1)_OBJ_FILES) $$($(1)_SRC_IN_AR_FILES) $(vecho) "AR $$@" $(Q) mkdir -p $$(dir $$@) $(Q) $(AR) cru $$@ $$^ diff --git a/parameters.mk b/parameters.mk index 8c6a8a3..67ab4dc 100644 --- a/parameters.mk +++ b/parameters.mk @@ -42,6 +42,9 @@ PRINTF_SCANF_FLOAT_SUPPORT ?= 1 FLAVOR ?= release # or debug +# Include source files into static library +INCLUDE_SRC_INTO_AR ?= 1 + # Compiler names, etc. assume gdb CROSS ?= xtensa-lx106-elf- diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..c8ecf8b --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,19 @@ +PROGRAM=tests + +PROGRAM_SRC_DIR = . ./cases + +# Add unity test framework headers & core source file +PROGRAM_INC_DIR = ./unity/src +PROGRAM_EXTRA_SRC_FILES = ./unity/src/unity.c + +TESTCASE_SRC_FILES = $(wildcard $(PROGRAM_DIR)cases/*.c) + +# Do not include source files into a static library because when adding this +# library with '--whole-archive' linker gives error that archive contains +# unknown objects (source files) +INCLUDE_SRC_INTO_AR = 0 + +# Link every object in the 'program' archive, to pick up constructor functions for test cases +EXTRA_LDFLAGS = -Wl,--whole-archive $(BUILD_DIR)program.a -Wl,--no-whole-archive + +include ../common.mk diff --git a/tests/cases/01_scheduler.c b/tests/cases/01_scheduler.c new file mode 100644 index 0000000..3949825 --- /dev/null +++ b/tests/cases/01_scheduler.c @@ -0,0 +1,71 @@ +#include "testcase.h" +#include +#include +#include + +/* Basic test cases to validate the FreeRTOS scheduler works */ + +DEFINE_SOLO_TESTCASE(01_scheduler_basic) +DEFINE_SOLO_TESTCASE(01_scheduler_priorities) + +void set_variable(void *pvParameters) +{ + bool *as_bool = (bool *)pvParameters; + *as_bool = true; + /* deliberately a busywait at the end, not vTaskSuspend, to test priorities */ + while(1) { } +} + +/* Really simple - do created tasks run? */ +static void a_01_scheduler_basic() +{ + volatile bool a = false, b = false, c = false; + printf("top of scheduler...\n"); + uart_flush_txfifo(0); + + xTaskCreate(set_variable, (signed char *)"set_a", 128, (void *)&a, tskIDLE_PRIORITY, NULL); + xTaskCreate(set_variable, (signed char *)"set_b", 128, (void *)&b, tskIDLE_PRIORITY, NULL); + xTaskCreate(set_variable, (signed char *)"set_c", 128, (void *)&c, tskIDLE_PRIORITY, NULL); + + TEST_ASSERT_FALSE_MESSAGE(a, "task set_a shouldn't run yet"); + TEST_ASSERT_FALSE_MESSAGE(b, "task set_b shouldn't run yet"); + TEST_ASSERT_FALSE_MESSAGE(c, "task set_c shouldn't run yet"); + + vTaskDelay(5); + + TEST_ASSERT_TRUE_MESSAGE(a, "task set_a should have run"); + TEST_ASSERT_TRUE_MESSAGE(b, "task set_b should have run"); + TEST_ASSERT_TRUE_MESSAGE(c, "task set_c should have run"); + TEST_PASS(); +} + +/* Verify that a high-priority task will starve a lower priority task */ +static void a_01_scheduler_priorities() +{ + /* Increase priority of the init task to make sure it always takes priority */ + vTaskPrioritySet(xTaskGetCurrentTaskHandle(), tskIDLE_PRIORITY+4); + + bool lower = false, higher = false; + xTaskHandle task_lower, task_higher; + + xTaskCreate(set_variable, (signed char *)"high_prio", 128, (void *)&higher, tskIDLE_PRIORITY+1, &task_higher); + xTaskCreate(set_variable, (signed char *)"low_prio", 128, (void *)&lower, tskIDLE_PRIORITY, &task_lower); + + TEST_ASSERT_FALSE_MESSAGE(higher, "higher prio task should not have run yet"); + TEST_ASSERT_FALSE_MESSAGE(lower, "lower prio task should not have run yet"); + + vTaskDelay(2); + + TEST_ASSERT_TRUE_MESSAGE(higher, "higher prio task should have run"); + TEST_ASSERT_FALSE_MESSAGE(lower, "lower prio task should not have run"); + + /* Bump lower priority task over higher priority task */ + vTaskPrioritySet(task_lower, tskIDLE_PRIORITY+2); + + TEST_ASSERT_FALSE_MESSAGE(lower, "lower prio task should still not have run yet"); + + vTaskDelay(1); + + TEST_ASSERT_TRUE_MESSAGE(lower, "lower prio task should have run"); + TEST_PASS(); +} diff --git a/tests/cases/02_heap.c b/tests/cases/02_heap.c new file mode 100644 index 0000000..54bb5e3 --- /dev/null +++ b/tests/cases/02_heap.c @@ -0,0 +1,56 @@ +#include "testcase.h" +#include +#include +#include + +DEFINE_SOLO_TESTCASE(02_heap_simple) +DEFINE_SOLO_TESTCASE(02_heap_full) + +/* Simple heap accounting tests */ +static void a_02_heap_simple() +{ + struct mallinfo info = mallinfo(); + printf("'arena' allocation size %d bytes\n", info.arena); + /* This is really a sanity check, if the "arena" size shrinks then + this is a good thing and we can update the test. If it grows + then we can also update the test, but we need a good reason. */ + TEST_ASSERT_INT_WITHIN_MESSAGE(1000, 15000, info.arena, "Initial allocated heap should be approximately 15kB. SEE COMMENT."); + + uint32_t freeheap = xPortGetFreeHeapSize(); + printf("xPortGetFreeHeapSize = %d bytes\n", freeheap); + TEST_ASSERT_TRUE_MESSAGE(freeheap > 20000, "Should be at least 20kB free."); + + uint8_t *buf = malloc(8192); + /* <-- have to do something with buf or gcc helpfully optimises it out! */ + memset(buf, 0xEE, 8192); + uint32_t after = xPortGetFreeHeapSize(); + struct mallinfo after_info = mallinfo(); + printf("after arena size = %d bytes\n", after_info.arena); + printf("after xPortGetFreeHeapSize = %d bytes\n", after); + TEST_ASSERT_UINT32_WITHIN_MESSAGE(100, info.arena+8192, after_info.arena, "Allocated heap 'after' size should be 8kB more than before"); + TEST_ASSERT_UINT32_WITHIN_MESSAGE(100, freeheap-8192, after, "Free heap size should be 8kB less than before"); + + free(buf); + after = xPortGetFreeHeapSize(); + printf("after freeing xPortGetFreeHeapSize = %d bytes\n", after); + TEST_ASSERT_UINT32_WITHIN_MESSAGE(100, freeheap, after, "Free heap size after freeing buffer should be close to initial"); + TEST_PASS(); +} + +/* Ensure malloc behaves when out of memory */ +static void a_02_heap_full() +{ + void *x = malloc(65536); + TEST_ASSERT_NULL_MESSAGE(x, "Allocating 64kB should fail and return null"); + + void *y = malloc(32768); + TEST_ASSERT_NOT_NULL_MESSAGE(y, "Allocating 32kB should succeed"); + + void *z = malloc(32768); + TEST_ASSERT_NULL_MESSAGE(z, "Allocating second 32kB should fail"); + + free(y); + z = malloc(32768); + TEST_ASSERT_NOT_NULL_MESSAGE(z, "Allocating 32kB should succeed after first block freed"); + TEST_PASS(); +} diff --git a/tests/cases/03_byte_load_flash.c b/tests/cases/03_byte_load_flash.c new file mode 100644 index 0000000..4554e4c --- /dev/null +++ b/tests/cases/03_byte_load_flash.c @@ -0,0 +1,480 @@ +/* Unit tests to verify the "unaligned load handler" in core/exception_vectors.S that allows us to + complete byte loads from unaligned memory, etc. + + Adapted from a test program in 'experiments' that did this. +*/ +#include "testcase.h" +#include "esp/rom.h" +#include "esp/timer.h" +#include "esp/uart.h" +#include "espressif/esp_common.h" +#include "xtensa_ops.h" +#include "FreeRTOS.h" +#include "task.h" +#include "queue.h" + +#include "string.h" +#include "strings.h" + +#include + +#define TESTSTRING "O hai there! %d %d %d" + +static char dramtest[] = TESTSTRING; + +static const __attribute__((section(".iram1.notrodata"))) + char iramtest[] = TESTSTRING; + +static const __attribute__((section(".text.notrodata"))) + char iromtest[] = TESTSTRING; + +static const volatile __attribute__((section(".iram1.notliterals"))) + int16_t unsigned_shorts[] = { -3, -4, -5, -32767, 44 }; + +static const __attribute__((section(".iram1.notrodata"))) + char sanity_test_data[] = { + 0x01, 0x55, 0x7e, 0x2a, 0x81, 0xd5, 0xfe, 0xaa + }; + +DEFINE_SOLO_TESTCASE(03_byte_load_verify_sections) + +#define PTR_IN_REGION(PTR, START, LEN) \ + ((START <= (intptr_t)(PTR)) && ((intptr_t)(PTR) < (START+LEN))) + +/* Sanity check, ensure the addresses of the various test strings + * are in the correct address space regions. */ +static void a_03_byte_load_verify_sections() +{ + printf("dramtest addr %p\n", dramtest); + TEST_ASSERT_MESSAGE(PTR_IN_REGION(dramtest, 0x3FFE8000, 0x14000), + "dramtest should be in DRAM region"); + + printf("iramtest addr %p\n", iramtest); + TEST_ASSERT_MESSAGE(PTR_IN_REGION(iramtest, 0x40100000, 0x8000), + "iramtest should be in IRAM region"); + + printf("iromtest addr %p\n", iromtest); + TEST_ASSERT_MESSAGE(PTR_IN_REGION(iromtest, 0x40202010, (0x100000 - 0x2010)), + "iromtest sohuld be in IROM region"); + + printf("unsigned_shorts addr %p\n", unsigned_shorts); + TEST_ASSERT_MESSAGE(PTR_IN_REGION(unsigned_shorts, 0x40100000, 0x8000), + "unsigned_shorts should be in IRAM region"); + + printf("sanity_test_data addr %p\n", sanity_test_data); + TEST_ASSERT_MESSAGE(PTR_IN_REGION(sanity_test_data, 0x40100000, 0x8000), + "sanity_test_data should be in IRAM region"); + + TEST_PASS(); +} + + +/* test utility functions used for '03_byte_load_test_strings' + + returns the expected string result */ +typedef const char *(* test_with_fn_t)(const char *string); + +static char buf[64]; + +static const char * test_memcpy_aligned(const char *string) +{ + memcpy(buf, string, 16); + return "O hai there! %d "; +} + +static const char * test_memcpy_unaligned(const char *string) +{ + memcpy(buf, string, 15); + return "O hai there! %d"; +} + + +static const char * test_memcpy_unaligned2(const char *string) +{ + memcpy(buf, string+1, 15); + return " hai there! %d "; +} + +static const char * test_strcpy(const char *string) +{ + strcpy(buf, string); + return dramtest; +} + +static const char * test_sprintf(const char *string) +{ + sprintf(buf, string, 1, 2, 3); + return "O hai there! 1 2 3"; +} + +static const char * test_sprintf_arg(const char *string) +{ + sprintf(buf, "%s", string); + return dramtest; +} + +static const char * test_naive_strcpy(const char *string) +{ + char *to = buf; + while((*to++ = *string++)) + ; + return dramtest; +} + +static const char * test_naive_strcpy_a0(const char *string) +{ + asm volatile ( +" mov a8, %0 \n" +" mov a9, %1 \n" +"tns_loop%=: l8ui a0, a9, 0 \n" +" addi.n a9, a9, 1 \n" +" s8i a0, a8, 0 \n" +" addi.n a8, a8, 1 \n" +" bnez a0, tns_loop%=\n" + : : "r" (buf), "r" (string) : "a0", "a8", "a9"); + return dramtest; +} + +static const char * test_naive_strcpy_a2(const char *string) +{ + asm volatile ( +" mov a8, %0 \n" +" mov a9, %1 \n" +"tns_loop%=: l8ui a2, a9, 0 \n" +" addi.n a9, a9, 1 \n" +" s8i a2, a8, 0 \n" +" addi.n a8, a8, 1 \n" +" bnez a2, tns_loop%=\n" + : : "r" (buf), "r" (string) : "a2", "a8", "a9"); + return dramtest; +} + +static const char * test_naive_strcpy_a3(const char *string) +{ + asm volatile ( +" mov a8, %0 \n" +" mov a9, %1 \n" +"tns_loop%=: l8ui a3, a9, 0 \n" +" addi.n a9, a9, 1 \n" +" s8i a3, a8, 0 \n" +" addi.n a8, a8, 1 \n" +" bnez a3, tns_loop%=\n" + : : "r" (buf), "r" (string) : "a3", "a8", "a9"); + return TESTSTRING; +} + +static const char * test_naive_strcpy_a4(const char *string) +{ + asm volatile ( +" mov a8, %0 \n" +" mov a9, %1 \n" +"tns_loop%=: l8ui a4, a9, 0 \n" +" addi.n a9, a9, 1 \n" +" s8i a4, a8, 0 \n" +" addi.n a8, a8, 1 \n" +" bnez a4, tns_loop%=\n" + : : "r" (buf), "r" (string) : "a4", "a8", "a9"); + return TESTSTRING; +} + +static const char * test_naive_strcpy_a5(const char *string) +{ + asm volatile ( +" mov a8, %0 \n" +" mov a9, %1 \n" +"tns_loop%=: l8ui a5, a9, 0 \n" +" addi.n a9, a9, 1 \n" +" s8i a5, a8, 0 \n" +" addi.n a8, a8, 1 \n" +" bnez a5, tns_loop%=\n" + : : "r" (buf), "r" (string) : "a5", "a8", "a9"); + return TESTSTRING; +} + +static const char * test_naive_strcpy_a6(const char *string) +{ + asm volatile ( +" mov a8, %0 \n" +" mov a9, %1 \n" +"tns_loop%=: l8ui a6, a9, 0 \n" +" addi.n a9, a9, 1 \n" +" s8i a6, a8, 0 \n" +" addi.n a8, a8, 1 \n" +" bnez a6, tns_loop%=\n" + : : "r" (buf), "r" (string) : "a6", "a8", "a9"); + return TESTSTRING; +} + +static const char * test_noop(const char *string) +{ + buf[0] = 0; + return ""; +} + +static uint32_t IRAM inner_string_test(const char *string, test_with_fn_t testfn, const char *testfn_label, uint32_t nullvalue, bool evict_cache) +{ + printf(" .. against %30s: ", testfn_label); + vPortEnterCritical(); + uint32_t before; + RSR(before, CCOUNT); + const int TEST_REPEATS = 1000; + for(int i = 0; i < TEST_REPEATS; i++) { + memset(buf, 0, sizeof(buf)); + const char *expected = testfn(string); + TEST_ASSERT_EQUAL_STRING_MESSAGE(expected, buf, testfn_label); + if(evict_cache) { + Cache_Read_Disable(); + Cache_Read_Enable(0,0,1); + } + } + uint32_t after; + RSR(after, CCOUNT); + vPortExitCritical(); + uint32_t instructions = (after-before)/TEST_REPEATS - nullvalue; + printf("%5d instructions\r\n", instructions); + return instructions; +} + +static void string_test(const char *string, char *label, bool evict_cache) +{ + printf("Testing %s (%p) '%s'\r\n", label, string, string); + printf("Formats as: '"); + printf(string, 1, 2, 3); + printf("'\r\n"); + uint32_t nullvalue = inner_string_test(string, test_noop, "null op", 0, evict_cache); + inner_string_test(string, test_memcpy_aligned, "memcpy - aligned len", nullvalue, evict_cache); + inner_string_test(string, test_memcpy_unaligned, "memcpy - unaligned len", nullvalue, evict_cache); + inner_string_test(string, test_memcpy_unaligned2, "memcpy - unaligned start&len", nullvalue, evict_cache); + inner_string_test(string, test_strcpy, "strcpy", nullvalue, evict_cache); + inner_string_test(string, test_naive_strcpy, "naive strcpy", nullvalue, evict_cache); + inner_string_test(string, test_naive_strcpy_a0, "naive strcpy (a0)", nullvalue, evict_cache); + inner_string_test(string, test_naive_strcpy_a2, "naive strcpy (a2)", nullvalue, evict_cache); + inner_string_test(string, test_naive_strcpy_a3, "naive strcpy (a3)", nullvalue, evict_cache); + inner_string_test(string, test_naive_strcpy_a4, "naive strcpy (a4)", nullvalue, evict_cache); + inner_string_test(string, test_naive_strcpy_a5, "naive strcpy (a5)", nullvalue, evict_cache); + inner_string_test(string, test_naive_strcpy_a6, "naive strcpy (a6)", nullvalue, evict_cache); + inner_string_test(string, test_sprintf, "sprintf", nullvalue, evict_cache); + inner_string_test(string, test_sprintf_arg, "sprintf format arg", nullvalue, evict_cache); +} + +DEFINE_SOLO_TESTCASE(03_byte_load_test_strings) + +/* Test various operations on strings in various regions */ +static void a_03_byte_load_test_strings() +{ + string_test(dramtest, "DRAM", 0); + string_test(iramtest, "IRAM", 0); + string_test(iromtest, "Cached flash", 0); + string_test(iromtest, "'Uncached' flash", 1); + TEST_PASS(); +} + +static volatile bool frc1_ran; +static volatile bool frc1_finished; +static volatile char frc1_buf[80]; + +DEFINE_SOLO_TESTCASE(03_byte_load_test_isr) + +static void frc1_interrupt_handler(void) +{ + frc1_ran = true; + timer_set_run(FRC1, false); + strcpy((char *)frc1_buf, iramtest); + frc1_finished = true; +} + +/* Verify that the unaligned loader can run inside an ISR */ +static void a_03_byte_load_test_isr() +{ + printf("Testing behaviour inside ISRs...\r\n"); + timer_set_interrupts(FRC1, false); + timer_set_run(FRC1, false); + _xt_isr_attach(INUM_TIMER_FRC1, frc1_interrupt_handler); + timer_set_frequency(FRC1, 1000); + timer_set_interrupts(FRC1, true); + timer_set_run(FRC1, true); + sdk_os_delay_us(2000); + + if(!frc1_ran) + TEST_FAIL_MESSAGE("ERROR: FRC1 timer exception never fired.\r\n"); + else if(!frc1_finished) + TEST_FAIL_MESSAGE("ERROR: FRC1 timer exception never finished.\r\n"); + else if(strcmp((char *)frc1_buf, iramtest)) + TEST_FAIL_MESSAGE("ERROR: FRC1 strcpy from IRAM failed.\r\n"); + else + TEST_PASS(); +} + +DEFINE_SOLO_TESTCASE(03_byte_load_test_sign_extension) + +static void a_03_byte_load_test_sign_extension() +{ + /* this step seems to be necessary so the compiler will actually generate l16si */ + int16_t *shorts_p = (int16_t *)unsigned_shorts; + if(shorts_p[0] == -3 && shorts_p[1] == -4 && shorts_p[2] == -5 && shorts_p[3] == -32767 && shorts_p[4] == 44) + { + TEST_PASS(); + } else { + sprintf(buf, "l16si sign extension failed. Got values %d %d %d %d %d\r\n", shorts_p[0], shorts_p[1], shorts_p[2], shorts_p[3], shorts_p[4]); + TEST_FAIL_MESSAGE(buf); + } +} + + +/* test that running unaligned loads in a running FreeRTOS system doesn't break things + + The following tests run inside a FreeRTOS task, after everything else. +*/ +DEFINE_SOLO_TESTCASE(03_byte_load_test_system_interaction); + +static void task_load_test_system_interaction() +{ + uint32_t start = xTaskGetTickCount(); + printf("Starting system/timer interaction test (takes approx 1 second)...\n"); + for(int i = 0; i < 5000; i++) { + test_naive_strcpy_a0(iromtest); + test_naive_strcpy_a2(iromtest); + test_naive_strcpy_a3(iromtest); + test_naive_strcpy_a4(iromtest); + test_naive_strcpy_a5(iromtest); + test_naive_strcpy_a6(iromtest); + /* + const volatile char *string = iromtest; + volatile char *to = dest; + while((*to++ = *string++)) + ; + */ + } + uint32_t ticks = xTaskGetTickCount() - start; + printf("Timer interaction test PASSED after %d ticks.\n", ticks); + TEST_PASS(); +} + +static void a_03_byte_load_test_system_interaction() +{ + xTaskCreate(task_load_test_system_interaction, (signed char *)"interactionTask", 256, NULL, 2, NULL); + while(1) { + vTaskDelay(100); + } +} + +/* The following "sanity tests" are designed to try to execute every code path + * of the LoadStoreError handler, with a variety of offsets and data values + * designed to catch any mask/shift errors, sign-extension bugs, etc */ +DEFINE_SOLO_TESTCASE(03_byte_load_test_sanity) + +/* (Contrary to expectations, 'mov a15, a15' in Xtensa is not technically a + * no-op, but is officially "undefined and reserved for future use", so we need + * a special case in the case where reg == "a15" so we don't end up generating + * those opcodes. GCC is smart enough to optimize away the whole conditional + * and just insert the correct asm block, since `reg` is a static argument.) */ +#define LOAD_VIA_REG(op, reg, addr, var) \ + if (strcmp(reg, "a15")) { \ + asm volatile ( \ + "mov a15, " reg "\n\t" \ + op " " reg ", %1, 0\n\t" \ + "mov %0, " reg "\n\t" \ + "mov " reg ", a15\n\t" \ + : "=r" (var) : "r" (addr) : "a15" ); \ + } else { \ + asm volatile ( \ + op " " reg ", %1, 0\n\t" \ + "mov %0, " reg "\n\t" \ + : "=r" (var) : "r" (addr) : "a15" ); \ + } + +#define TEST_LOAD(op, reg, addr, value) \ + { \ + int32_t result; \ + LOAD_VIA_REG(op, reg, addr, result); \ + if (result != value) sanity_test_failed(op, reg, addr, value, result); \ + } + +static void sanity_test_failed(const char *testname, const char *reg, const void *addr, int32_t value, int32_t result) { + uint32_t actual_data = *(uint32_t *)((uint32_t)addr & 0xfffffffc); + sprintf(buf, "%s %s from %p (32-bit value: 0x%x): Expected 0x%08x (%d), got 0x%08x (%d)\n", testname, reg, addr, actual_data, value, value, result, result); + TEST_FAIL_MESSAGE(buf); +} + +static void sanity_test_l8ui(const void *addr, int32_t value) { + TEST_LOAD("l8ui", "a0", addr, value); + TEST_LOAD("l8ui", "a1", addr, value); + TEST_LOAD("l8ui", "a2", addr, value); + TEST_LOAD("l8ui", "a3", addr, value); + TEST_LOAD("l8ui", "a4", addr, value); + TEST_LOAD("l8ui", "a5", addr, value); + TEST_LOAD("l8ui", "a6", addr, value); + TEST_LOAD("l8ui", "a7", addr, value); + TEST_LOAD("l8ui", "a8", addr, value); + TEST_LOAD("l8ui", "a9", addr, value); + TEST_LOAD("l8ui", "a10", addr, value); + TEST_LOAD("l8ui", "a11", addr, value); + TEST_LOAD("l8ui", "a12", addr, value); + TEST_LOAD("l8ui", "a13", addr, value); + TEST_LOAD("l8ui", "a14", addr, value); + TEST_LOAD("l8ui", "a15", addr, value); +} + +static void sanity_test_l16ui(const void *addr, int32_t value) { + TEST_LOAD("l16ui", "a0", addr, value); + TEST_LOAD("l16ui", "a1", addr, value); + TEST_LOAD("l16ui", "a2", addr, value); + TEST_LOAD("l16ui", "a3", addr, value); + TEST_LOAD("l16ui", "a4", addr, value); + TEST_LOAD("l16ui", "a5", addr, value); + TEST_LOAD("l16ui", "a6", addr, value); + TEST_LOAD("l16ui", "a7", addr, value); + TEST_LOAD("l16ui", "a8", addr, value); + TEST_LOAD("l16ui", "a9", addr, value); + TEST_LOAD("l16ui", "a10", addr, value); + TEST_LOAD("l16ui", "a11", addr, value); + TEST_LOAD("l16ui", "a12", addr, value); + TEST_LOAD("l16ui", "a13", addr, value); + TEST_LOAD("l16ui", "a14", addr, value); + TEST_LOAD("l16ui", "a15", addr, value); +} + +static void sanity_test_l16si(const void *addr, int32_t value) { + TEST_LOAD("l16si", "a0", addr, value); + TEST_LOAD("l16si", "a1", addr, value); + TEST_LOAD("l16si", "a2", addr, value); + TEST_LOAD("l16si", "a3", addr, value); + TEST_LOAD("l16si", "a4", addr, value); + TEST_LOAD("l16si", "a5", addr, value); + TEST_LOAD("l16si", "a6", addr, value); + TEST_LOAD("l16si", "a7", addr, value); + TEST_LOAD("l16si", "a8", addr, value); + TEST_LOAD("l16si", "a9", addr, value); + TEST_LOAD("l16si", "a10", addr, value); + TEST_LOAD("l16si", "a11", addr, value); + TEST_LOAD("l16si", "a12", addr, value); + TEST_LOAD("l16si", "a13", addr, value); + TEST_LOAD("l16si", "a14", addr, value); + TEST_LOAD("l16si", "a15", addr, value); +} + +static void a_03_byte_load_test_sanity(void) { + printf("== Performing sanity tests (sanity_test_data @ %p)...\n", sanity_test_data); + + sanity_test_l8ui(sanity_test_data + 0, 0x01); + sanity_test_l8ui(sanity_test_data + 1, 0x55); + sanity_test_l8ui(sanity_test_data + 2, 0x7e); + sanity_test_l8ui(sanity_test_data + 3, 0x2a); + sanity_test_l8ui(sanity_test_data + 4, 0x81); + sanity_test_l8ui(sanity_test_data + 5, 0xd5); + sanity_test_l8ui(sanity_test_data + 6, 0xfe); + sanity_test_l8ui(sanity_test_data + 7, 0xaa); + + sanity_test_l16ui(sanity_test_data + 0, 0x5501); + sanity_test_l16ui(sanity_test_data + 2, 0x2a7e); + sanity_test_l16ui(sanity_test_data + 4, 0xd581); + sanity_test_l16ui(sanity_test_data + 6, 0xaafe); + + sanity_test_l16si(sanity_test_data + 0, 0x5501); + sanity_test_l16si(sanity_test_data + 2, 0x2a7e); + sanity_test_l16si(sanity_test_data + 4, -10879); + sanity_test_l16si(sanity_test_data + 6, -21762); + + printf("== Sanity tests completed.\n"); + TEST_PASS(); +} diff --git a/tests/include/testcase.h b/tests/include/testcase.h new file mode 100644 index 0000000..63e91de --- /dev/null +++ b/tests/include/testcase.h @@ -0,0 +1,59 @@ +#ifndef _TESTCASE_H +#define _TESTCASE_H +#include +#include +#include "esp/uart.h" + +/* Unity is the framework with test assertions, etc. */ +#include "unity.h" + +/* Need to explicitly flag once a test has completed successfully. */ +#undef TEST_PASS +#define TEST_PASS() do { UnityConcludeTest(); while(1) { } } while (0) + +/* Types of test, defined by hardware requirements */ +typedef enum { + SOLO, /* Test require "ESP A" only, no other connections */ + DUAL, /* Test requires "ESP A" and "ESP "B", basic interconnections between them */ + EYORE_TEST, /* Test requires an eyore-test board with onboard STM32F0 */ +} testcase_type_t; + +typedef void (testcase_fn_t)(void); + +typedef struct { + const char *name; + const char *file; + int line; + testcase_type_t type; + testcase_fn_t *a_fn; + testcase_fn_t *b_fn; +} testcase_t; + +void testcase_register(const testcase_t *testcase); + +/* Register a test case using these macros. Use DEFINE_SOLO_TESTCASE for single-MCU tests, + and DEFINE_TESTCASE for all other test types. +*/ +#define DEFINE_SOLO_TESTCASE(NAME) \ + static testcase_fn_t a_##NAME; \ + _DEFINE_TESTCASE_COMMON(NAME, SOLO, a_##NAME, 0) + +#define DEFINE_TESTCASE(NAME, TYPE) \ + static testcase_fn_t a_##NAME; \ + static testcase_fn_t b_##NAME; \ + _DEFINE_TESTCASE_COMMON(NAME, TYPE, A_##NAME, B_##NAME) + + +#define _DEFINE_TESTCASE_COMMON(NAME, TYPE, A_FN, B_FN) \ + void __attribute__((constructor)) testcase_ctor_##NAME() { \ + const testcase_t testcase = { .name = #NAME, \ + .file = __FILE__, \ + .line = __LINE__, \ + .type = TYPE, \ + .a_fn = A_FN, \ + .b_fn = B_FN, \ + }; \ + testcase_register(&testcase); \ + } + +#endif diff --git a/tests/test_main.c b/tests/test_main.c new file mode 100644 index 0000000..a93d6e1 --- /dev/null +++ b/tests/test_main.c @@ -0,0 +1,115 @@ +#include "testcase.h" +#include +#include +#include + +/* Convert requirement enum to a string we can print */ +static const char *get_requirements_name(const testcase_type_t arg) { + switch(arg) { + case SOLO: + return "SOLO"; + case DUAL: + return "DUAL"; + case EYORE_TEST: + return "EYORE_TEST"; + default: + return "UNKNOWN"; + } +} + +static testcase_t *testcases; +static uint32_t testcases_count; +static uint32_t testcases_alloc; + +void testcase_register(const testcase_t *testcase) +{ + /* Grow the testcases buffer to fit the new test case, + this buffer will be freed before the test runs. + */ + if(testcases_count == testcases_alloc) { + testcases_alloc += 1; + testcases = realloc(testcases, testcases_alloc * sizeof(testcase_t)); + if(!testcases) { + printf("Failed to reallocate test case register length %d\n", testcases_alloc); + testcases_count = 0; + testcases_alloc = 0; + } + } + memcpy(&testcases[testcases_count++], testcase, sizeof(testcase_t)); +} + +void user_init(void) +{ + uart_set_baud(0, 115200); + printf("esp-open-rtos test runner.\n"); + printf("%d test cases are defined:\n\n", testcases_count); + for(int i = 0; i < testcases_count; i++) { + printf("CASE %d = %s %s\n", i, testcases[i].name, get_requirements_name(testcases[i].type)); + } + + printf("Enter A or B then number of test case to run, ie A0.\n"); + int case_idx = -1; + char type; + do { + printf("> "); + uart_rxfifo_wait(0,1); + type = uart_getc(0); + if(type != 'a' && type != 'A' && type != 'b' && type != 'B') { + printf("Type must be A or B.\n"); + continue; + } + + char idx_buf[6]; + for(int c = 0; c < sizeof(idx_buf); c++) { + uart_rxfifo_wait(0,1); + idx_buf[c] = uart_getc(0); + if(idx_buf[c] == ' ') { /* Ignore spaces */ + c--; + continue; + } + if(idx_buf[c] == '\r' || idx_buf[c] == '\n') { + idx_buf[c] = 0; + case_idx = atoi(idx_buf); + break; + } + else if(idx_buf[c] < '0' || idx_buf[c] > '9') { + break; + } + } + + if(case_idx == -1) { + printf("Invalid case index"); + } + else if(case_idx < 0 || case_idx >= testcases_count) { + printf("Test case index out of range.\n"); + } + else if((type == 'b' || type =='B') && testcases[case_idx].type == SOLO) { + printf("No ESP type B for 'SOLO' test cases.\n"); + } else { + break; + } + } while(1); + if(type =='a') + type = 'A'; + else if (type == 'b') + type = 'B'; + testcase_t testcase; + memcpy(&testcase, &testcases[case_idx], sizeof(testcase_t)); + /* Free the register of test cases now we have the one we're running */ + free(testcases); + testcases_alloc = 0; + testcases_count = 0; + + printf("\nRunning test case %d (%s %s) as instance %c \nDefinition at %s:%d\n***\n", case_idx, + testcase.name, get_requirements_name(testcase.type), type, + testcase.file, testcase.line); + Unity.CurrentTestName = testcase.name; + Unity.TestFile = testcase.file; + Unity.CurrentTestLineNumber = testcase.line; + Unity.NumberOfTests = 1; + if(type=='A') + testcase.a_fn(); + else + testcase.b_fn(); + TEST_FAIL_MESSAGE("\n\nTest initialisation routine returned without calling TEST_PASS. Buggy test?"); +} diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100755 index 0000000..4acd017 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +import sys +import argparse +import subprocess +import os +import serial +import threading +import re +import time +import traceback + +SHORT_OUTPUT_TIMEOUT=0.25 # timeout for resetting and/or waiting for more lines of output +TESTCASE_TIMEOUT=10 +TESTRUNNER_BANNER="esp-open-rtos test runner." + +def main(): + global verbose + args = parse_args() + verbose = args.verbose + + if not args.no_flash: + flash_image(args.aport) + if args.type != 'solo': + flash_image(args.bport) + + env = TestEnvironment(args.aport, TestEnvironment.A) + cases = env.get_testlist() + if args.type != 'solo': + env_b = TestEnvironment(args.bport, TestEnvironment.B) + cases_b = env_b.get_testlist() + if cases != cases_b: + raise TestRunnerError("Test cases on units A & B don't match") + + counts = dict((status,0) for status in TestResult.STATUS_NAMES.keys()) + failures = False + for test in cases: + res = test.run(env) + counts[res.status] += 1 + failures = failures or res.is_failure() + + print("%20s: %d" % ("Total tests", sum(c for c in counts.values()))) + print() + # print status counts for tests + for c in sorted(counts.keys()): + print("%20s: %d" % (TestResult.STATUS_NAMES[c], counts[c])) + + sys.exit(1 if failures else 0) + +class TestCase(object): + def __init__(self, index, name, case_type): + self.name = name + self.index = index + self.case_type = case_type + + def __repr__(self): + return "#%d: %s (%s)" % (self.index, self.name, self.case_type) + + def __eq__(self, other): + return (self.index == other.index + and self.name == other.name + and self.case_type == other.case_type) + + def run(self, env_a, env_b = None): + """ + Run the test represented by this instance, against the environment(s) passed in. + + Returns a TestResult + """ + sys.stdout.write("Running test case '%s'...%s" % (self.name, "\n" if verbose else " "*(40-len(self.name)))) + mon_a = env_a.start_testcase(self) + mon_b = env_b.start_testcase(self) if env_b else None + while True: + if mon_a.get_result() and (mon_b is None or mon_b.get_result()): + break # all running test environments have finished + + # or, in the case both are running, stop as soon as either environemnt shows a failure + try: + if mon_a.get_result().is_failure(): + mon_b.cancel() + break + except AttributeError: + pass + try: + if mon_b.get_result().is_failure(): + mon_a.cancel() + break + except AttributeError: + pass + time.sleep(0.1) + + if mon_b is not None: + # return whichever result is more severe + res = max(mon_a.get_result(), mon_b.get_result()) + else: + res = mon_a.get_result() + if not verbose: # finish the line after the ... + print(TestResult.STATUS_NAMES[res.status]) + if res.is_failure(): + message = res.message + if "/" in res.message: # cut anything before the file name in the failure + message = message[message.index("/"):] + print("FAILURE MESSAGE:\n%s\n" % message) + return res + +class TestResult(object): + """ Class to wrap a test result code and a message """ + # Test status flags, higher = more severe + CANCELLED = 0 + SKIPPED = 1 + PASSED = 2 + FAILED = 3 + ERROR = 4 + + STATUS_NAMES = { + CANCELLED : "Cancelled", + SKIPPED : "Skipped", + PASSED : "Passed", + FAILED : "Failed", + ERROR : "Error" + } + + def __init__(self, status, message): + self.status = status + self.message = message + + def is_failure(self): + return self.status >= TestResult.FAILED + + def __cmp__(self, other): + if other is None: + return 1 + return self.status - other.status + +class TestMonitor(object): + """ Class to monitor a running test case in a separate thread, defer reporting of the result until it's done. + + Can poll for completion by calling is_done(), read a TestResult via .get_result() + """ + def __init__(self, port, instance): + super(TestMonitor, self).__init__() + self._thread = threading.Thread(target=self._monitorThread) + self._port = port + self._instance = instance + self._result = None + self._cancelled = False + self.output = "" + self._thread.start() + + def cancel(self): + self._cancelled = True + + def is_done(self): + return self._result is not None + + def get_result(self): + return self._result + + def _monitorThread(self): + self.output = "" + start_time = time.time() + self._port.timeout = SHORT_OUTPUT_TIMEOUT + try: + while not self._cancelled and time.time() < start_time + TESTCASE_TIMEOUT: + line = self._port.readline().decode("utf-8", "ignore") + if line == "": + continue # timed out + self.output += "%s+%4.2fs %s" % (self._instance, time.time()-start_time, line) + verbose_print(line.strip()) + if line.endswith(":PASS\r\n"): + self._result = TestResult(TestResult.PASSED, "Test passed.") + return + elif ":FAIL:" in line: + self._result = TestResult(TestResult.FAILED, line) + return + elif line == TESTRUNNER_BANNER: + self._result = TestResult(TestResult.ERROR, "Test caused crash and reset.") + return + if not self._cancelled: + self._result = TestResult(TestResult.CANCELLED, "Cancelled") + else: + self._result = TestResult(TestResult.ERROR, "Test timed out") + + finally: + self._port.timeout = None + +class TestEnvironment(object): + A = "A" + B = "B" + + def __init__(self, port, instance): + self._name = port + self._port = TestSerialPort(port, baudrate=115200) + self._instance = instance + + def reset(self): + """ Resets the test board, and waits for the test runner program to start up """ + self._port.setDTR(False) + self._port.setRTS(True) + time.sleep(0.05) + self._port.flushInput() + self._port.setRTS(False) + verbose_print("Waiting for test runner startup...") + if not self._port.wait_line(lambda line: line == TESTRUNNER_BANNER): + raise TestRunnerError("Port %s failed to start test runner" % self._port) + + def get_testlist(self): + """ Resets the test board and returns the enumerated list of all supported tests """ + self.reset() + tests = [] + verbose_print("Enumerating tests...") + + def collect_testcases(line): + if line.startswith(">"): + return True # prompt means list of test cases is done, success + m = re.match(r"CASE (\d+) = (.+?) ([A-Z]+)", line) + if m is not None: + t = TestCase(int(m.group(1)), m.group(2), m.group(3).lower()) + verbose_print(t) + tests.append(t) + if not self._port.wait_line(collect_testcases): + raise TestRunnerError("Port %s failed to read test list" % self._port) + verbose_print("Port %s found %d test cases" % (self._name, len(tests))) + return tests + + def start_testcase(self, case): + """ Starts the specified test instance and returns an TestMonitor reader thread instance to monitor the output """ + # synchronously start the test case + self.reset() + if not self._port.wait_line(lambda line: line.startswith(">")): + raise TestRunnerError("Failed to read test runnner prompt") + command = "%s%d\r\n" % (self._instance, case.index) + self._port.write(command.encode("utf-8")) + return TestMonitor(self._port, self._instance) + + +def get_testdir(): + """ + Return the 'tests' directory in the source tree + (assuming the test_runner.py script is in that directory. + """ + res = os.path.dirname(__name__) + return "." if res == "" else res + + +def flash_image(serial_port): + # Bit hacky: rather than calling esptool directly, just use the Makefile flash target + # with the correct ESPPORT argument + env = dict(os.environ) + env["ESPPORT"] = serial_port + verbose_print("Building and flashing test image to %s..." % serial_port) + try: + stdout = sys.stdout if verbose else None + output = subprocess.run(["make","flash"], check=True, cwd=get_testdir(), stdout=stdout, stderr=subprocess.STDOUT, env=env) + except subprocess.CalledProcessError as e: + raise TestRunnerError("'make flash EPPORT=%s' failed with exit code %d" % (serial_port, e.returncode)) + verbose_print("Flashing successful.") + + +def parse_args(): + parser = argparse.ArgumentParser(description='esp-open-rtos testrunner', prog='test_runner') + + parser.add_argument( + '--type', '-t', + help='Type of test hardware attached to serial ports A & (optionally) B', + choices=['solo','dual','eyore_test'], default='solo') + + parser.add_argument( + '--aport', '-a', + help='Serial port for device A', + default='/dev/ttyUSB0') + + parser.add_argument( + '--bport', '-b', + help='Serial port for device B (ignored if type is \'solo\')', + default='/dev/ttyUSB1') + + parser.add_argument( + '--no-flash', '-n', + help='Don\'t flash the test binary image before running tests', + action='store_true', + default=False) + + parser.add_argument( + '--verbose', '-v', + help='Verbose test runner debugging output', + action='store_true', + default=False) + + parser.add_argument('testcases', nargs='*', + help='Optional list of test cases to run. By default, all tests are run.') + + return parser.parse_args() + + +class TestRunnerError(RuntimeError): + def __init__(self, message): + RuntimeError.__init__(self, message) + +class TestSerialPort(serial.Serial): + def __init__(self, *args, **kwargs): + super(TestSerialPort, self).__init__(*args, **kwargs) + + def wait_line(self, callback, timeout = SHORT_OUTPUT_TIMEOUT): + """ Wait for the port to output a particular piece of line content, as judged by callback + + Callback called as 'callback(line)' and returns not-True if non-match otherwise can return any value. + + Returns first non-False result from the callback, or None if it timed out waiting for a new line. + + Note that a serial port spewing legitimate lines of output may block this function forever, if callback + doesn't detect this is happening. + """ + self.timeout = timeout + try: + res = None + while not res: + line = self.readline() + if line == b"": + break # timed out + line = line.decode("utf-8", "ignore").rstrip() + res = callback(line) + return res + finally: + self.timeout = None + +verbose = False + + +def verbose_print(msg): + if verbose: + print(msg) + +if __name__ == '__main__': + try: + main() + except TestRunnerError as e: + print(e) + sys.exit(2) + diff --git a/tests/unity b/tests/unity new file mode 160000 index 0000000..bbf2fe3 --- /dev/null +++ b/tests/unity @@ -0,0 +1 @@ +Subproject commit bbf2fe3a934f96cd00693841247a689e57a17b0d