Move unaligned_load 'experiment' to set of automated tests

(Test case registration buffer gets freed before the test case runs.)

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "extras/mbedtls/mbedtls"]
 	path = extras/mbedtls/mbedtls
 	url = https://github.com/ARMmbed/mbedtls.git
+[submodule "tests/unity"]
+	path = tests/unity
+	url = https://github.com/ThrowTheSwitch/Unity.git

FreeRTOS/Source/portable/esp8266/port.c

@@ -81,13 +81,14 @@
 unsigned cpu_sr;
 char level1_int_disabled;
 
-/* Supervisor stack pointer entry. This is the "high water mark" of how far the
-   supervisor stack grew down before task started.
+/* Supervisor stack pointer entry. This is the "high water mark" of
+   how far the supervisor stack grew down before task started. Is zero
+   before the scheduler starts.
 
-   After tasks start, task stacks are all allocated from the heap and
-   FreeRTOS checks for stack overflow.
+ After the scheduler starts, task stacks are all allocated from the
+ heap and FreeRTOS checks for stack overflow.
 */
-static uint32_t xPortSupervisorStackPointer;
+void *xPortSupervisorStackPointer;
 
 /*
  * Stack initialization
@@ -178,9 +179,6 @@ void xPortSysTickHandle (void)
 	//OpenNMI();
 }
 
-static bool sdk_compat_initialised;
-void sdk_compat_initialise(void);
-
 /*
  * See header file for description.
  */
@@ -189,14 +187,6 @@ portBASE_TYPE xPortStartScheduler( void )
     _xt_isr_attach(INUM_SOFT, SV_ISR);
     _xt_isr_unmask(BIT(INUM_SOFT));
 
-    /* ENORMOUS HACK: Call the sdk_compat_initialise() function.
-       This can be removed happily once we have open source startup code.
-    */
-    if(!sdk_compat_initialised) {
-        sdk_compat_initialised = true;
-        sdk_compat_initialise();
-    }
-
     /* Initialize system tick timer interrupt and schedule the first tick. */
     _xt_isr_attach(INUM_TICK, sdk__xt_timer_int);
     _xt_isr_unmask(BIT(INUM_TICK));
@@ -229,7 +219,7 @@ size_t xPortGetFreeHeapSize( void )
     struct mallinfo mi = mallinfo();
     uint32_t brk_val = (uint32_t) sbrk(0);
 
-    uint32_t sp = xPortSupervisorStackPointer;
+    intptr_t sp = (intptr_t)xPortSupervisorStackPointer;
     if(sp == 0) /* scheduler not started */
         __asm__ __volatile__ ("mov %0, a1\n" : "=a"(sp));
     return sp - brk_val + mi.fordblks;

common.mk

@@ -109,7 +109,7 @@ CXXFLAGS	?= $(C_CXX_FLAGS) -fno-exceptions -fno-rtti $(EXTRA_CXXFLAGS)
 # these aren't technically preprocesor args, but used by all 3 of C, C++, assembler
 CPPFLAGS	+= -mlongcalls -mtext-section-literals
 
-LDFLAGS		= -nostdlib -Wl,--no-check-sections -L$(BUILD_DIR)sdklib -L$(ROOT)lib -u $(ENTRY_SYMBOL) -Wl,-static -Wl,-Map=build/${PROGRAM}.map  $(EXTRA_LDFLAGS)
+LDFLAGS		= -nostdlib -Wl,--no-check-sections -L$(BUILD_DIR)sdklib -L$(ROOT)lib -u $(ENTRY_SYMBOL) -Wl,-static -Wl,-Map=build/${PROGRAM}.map $(addprefix -T,$(LINKER_SCRIPTS)) $(EXTRA_LDFLAGS)
 
 ifeq ($(SPLIT_SECTIONS),1)
   C_CXX_FLAGS += -ffunction-sections -fdata-sections
@@ -139,7 +139,7 @@ LINKER_SCRIPTS  = $(ROOT)ld/nonota.ld
 else
 LINKER_SCRIPTS = $(ROOT)ld/ota.ld
 endif
-LINKER_SCRIPTS += $(ROOT)ld/common.ld $(ROOT)ld/rom.ld
+LINKER_SCRIPTS += $(ROOT)ld/common.ld $(ROOT)ld/rom.ld $(EXTRA_LINKER_SCRIPTS)
 
 ####
 #### no user configurable options below here
@@ -163,7 +163,6 @@ PROGRAM_DIR := $(dir $(firstword $(MAKEFILE_LIST)))
 SDK_LIB_ARGS  = $(addprefix -l,$(SDK_LIBS))
 LIB_ARGS      = $(addprefix -l,$(LIBS))
 PROGRAM_OUT   = $(BUILD_DIR)$(PROGRAM).out
-LDFLAGS      += $(addprefix -T,$(LINKER_SCRIPTS))
 
 ifeq ($(OTA),0)
 # for non-OTA, we create two different files for uploading into the flash
@@ -280,7 +279,7 @@ $$($(1)_OBJ_DIR)%.o: $$($(1)_REAL_ROOT)%.S $$($(1)_MAKEFILE) $(wildcard $(ROOT)*
 $$($(1)_AR): $$($(1)_OBJ_FILES) $$($(1)_SRC_FILES)
 	$(vecho) "AR $$@"
 	$(Q) mkdir -p $$(dir $$@)
-	$(Q) $(AR) cru $$@ $$^
+	$(Q) $(AR) cru $$@ $$(filter %.o,$$^)
 
 COMPONENT_ARS += $$($(1)_AR)
 

core/app_main.c

@@ -407,6 +407,9 @@ void sdk_user_init_task(void *params) {
     vTaskDelete(NULL);
 }
 
+extern void (*__init_array_start)(void);
+extern void (*__init_array_end)(void);
+
 // .Lfunc009 -- .irom0.text+0x5b4
 static void user_start_phase2(void) {
     uint8_t *buf;
@@ -437,6 +440,12 @@ static void user_start_phase2(void) {
     uart_flush_txfifo(0);
     uart_flush_txfifo(1);
 
+    // Call gcc constructor functions
+    void (**ctor)(void);
+    for ( ctor = &__init_array_start; ctor != &__init_array_end; ++ctor) {
+        (*ctor)();
+    }
+
     if (phy_info[0] != 5) {
         // Bad version byte.  Discard what we read and use default values
         // instead.

core/newlib_syscalls.c

@@ -12,6 +12,8 @@
 #include <esp/uart.h>
 #include <stdlib.h>
 
+extern void *xPortSupervisorStackPointer;
+
 IRAM caddr_t _sbrk_r (struct _reent *r, int incr)
 {
     extern char   _heap_start; /* linker script defined */
@@ -21,13 +23,16 @@ IRAM caddr_t _sbrk_r (struct _reent *r, int incr)
     if (heap_end == NULL)
 	heap_end = &_heap_start;
     prev_heap_end = heap_end;
-    /* TODO: Check stack collision
-       if (heap_end + incr > stack_ptr)
+
+    intptr_t sp = (intptr_t)xPortSupervisorStackPointer;
+    if(sp == 0) /* scheduler not started */
+        __asm__ __volatile__ ("mov %0, a1\n" : "=a"(sp));
+
+    if ((intptr_t)heap_end + incr >= sp)
     {
-       _write (1, "_sbrk: Heap collided with stack\n", 32);
-       while(1) {}
+        r->_errno = ENOMEM;
+        return (caddr_t)-1;
     }
-    */
     heap_end += incr;
 
     return (caddr_t) prev_heap_end;

core/sdk_compat.c

@@ -17,20 +17,6 @@ void IRAM *zalloc(size_t nbytes)
     return calloc(1, nbytes);
 }
 
-extern void (*__init_array_start)(void);
-extern void (*__init_array_end)(void);
-
-/* Do things which should be done as part of the SDK startup code, but aren't.
-   TODO: Move into app_main.c
-*/
-void sdk_compat_initialise()
-{
-    /* Call C++ constructors or C functions marked with __attribute__((constructor)) */
-    void (**p)(void);
-    for ( p = &__init_array_start; p != &__init_array_end; ++p)
-        (*p)();
-}
-
 /* UART RX function from Espressif SDK internals.
  *
  * Not part of published API.

examples/experiments/unaligned_load/Makefile

@@ -1,2 +0,0 @@
-PROGRAM=unaligned_load
-include ../../../common.mk

ld/common.ld

@@ -139,13 +139,34 @@ SECTIONS
        (except for libgcc which is matched above.)
     */
     *(.literal .text .literal.* .text.*)
+    /* Anything explicitly marked as "irom" or "irom0" should go here */
+    *(.irom.* .irom.*.* .irom0.*)
+    _irom0_text_end = ABSOLUTE(.);
+
+    /* Temporary .rodata hacks start here, eventually all rodata will
+    be in irom by default */
     /* mbedtls rodata */
     *mbedtls.a:*.o(.rodata.* .rodata)
     /* actual certificate in example (TEMPORARY HACK) */
     *:cert.o(.rodata.* .rodata)
-    /* Anything explicitly marked as "irom" or "irom0" should go here */
-    *(.irom.* .irom.*.* .irom0.*)
-    _irom0_text_end = ABSOLUTE(.);
+    /*  C++ constructor and destructor tables, properly ordered:  */
+    __init_array_start = ABSOLUTE(.);
+    KEEP (*crtbegin.o(.ctors))
+    KEEP (*(EXCLUDE_FILE (*crtend.o) .ctors))
+    KEEP (*(SORT(.ctors.*)))
+    KEEP (*(.ctors))
+    __init_array_end = ABSOLUTE(.);
+    KEEP (*crtbegin.o(.dtors))
+    KEEP (*(EXCLUDE_FILE (*crtend.o) .dtors))
+    KEEP (*(SORT(.dtors.*)))
+    KEEP (*(.dtors))
+    /*  C++ exception handlers table:  */
+    __XT_EXCEPTION_DESCS__ = ABSOLUTE(.);
+    *(.xt_except_desc)
+    *(.gnu.linkonce.h.*)
+    __XT_EXCEPTION_DESCS_END__ = ABSOLUTE(.);
+    *(.xt_except_desc_end)
+
   } >irom0_0_seg :irom0_0_phdr
 
   .data : ALIGN(4)
@@ -179,23 +200,6 @@ SECTIONS
     *(.gnu.version_r)
     *(.eh_frame)
     . = (. + 3) & ~ 3;
-    /*  C++ constructor and destructor tables, properly ordered:  */
-    __init_array_start = ABSOLUTE(.);
-    KEEP (*crtbegin.o(.ctors))
-    KEEP (*(EXCLUDE_FILE (*crtend.o) .ctors))
-    KEEP (*(SORT(.ctors.*)))
-    KEEP (*(.ctors))
-    __init_array_end = ABSOLUTE(.);
-    KEEP (*crtbegin.o(.dtors))
-    KEEP (*(EXCLUDE_FILE (*crtend.o) .dtors))
-    KEEP (*(SORT(.dtors.*)))
-    KEEP (*(.dtors))
-    /*  C++ exception handlers table:  */
-    __XT_EXCEPTION_DESCS__ = ABSOLUTE(.);
-    *(.xt_except_desc)
-    *(.gnu.linkonce.h.*)
-    __XT_EXCEPTION_DESCS_END__ = ABSOLUTE(.);
-    *(.xt_except_desc_end)
     *(.dynamic)
     *(.gnu.version_d)
     . = ALIGN(4);		/* this table MUST be 4-byte aligned */
@@ -228,8 +232,7 @@ SECTIONS
     _heap_start = ABSOLUTE(.);
 /*    _stack_sentry = ALIGN(0x8); */
   } >dram0_0_seg :dram0_0_bss_phdr
-/* __stack = 0x3ffc8000; */
-
+/* __stack = 0x3ffc8000; <-- this value seems a bit odd, stack on sdk_user_start is ~0x3ffffce9 */
 
   .lit4 : ALIGN(4)
   {

tests/Makefile

@@ -0,0 +1,14 @@
+PROGRAM=tests
+
+PROGRAM_SRC_DIR = $(PROGRAM_DIR) $(PROGRAM_DIR)cases
+
+# Add unity test framework headers & core source file
+PROGRAM_INC_DIR = $(PROGRAM_DIR)unity/src
+PROGRAM_EXTRA_SRC_FILES = $(PROGRAM_DIR)unity/src/unity.c
+
+TESTCASE_SRC_FILES = $(wildcard $(PROGRAM_DIR)cases/*.c)
+
+# Link every object in the 'program' archive, to pick up constructor functions for test cases
+EXTRA_LDFLAGS = -Wl,--whole-archive $(BUILD_DIR)program.a -Wl,--no-whole-archive
+
+include ../common.mk

tests/cases/01_scheduler.c

@@ -0,0 +1,71 @@
+#include "testcase.h"
+#include <FreeRTOS.h>
+#include <task.h>
+#include <esp/uart.h>
+
+/* Basic test cases to validate the FreeRTOS scheduler works */
+
+DEFINE_SOLO_TESTCASE(01_scheduler_basic)
+DEFINE_SOLO_TESTCASE(01_scheduler_priorities)
+
+void set_variable(void *pvParameters)
+{
+    bool *as_bool = (bool *)pvParameters;
+    *as_bool = true;
+    /* deliberately a busywait at the end, not vTaskSuspend, to test priorities */
+    while(1) { }
+}
+
+/* Really simple - do created tasks run? */
+static void a_01_scheduler_basic()
+{
+    volatile bool a = false, b = false, c = false;
+    printf("top of scheduler...\n");
+    uart_flush_txfifo(0);
+
+    xTaskCreate(set_variable, (signed char *)"set_a", 128, (void *)&a, tskIDLE_PRIORITY, NULL);
+    xTaskCreate(set_variable, (signed char *)"set_b", 128, (void *)&b, tskIDLE_PRIORITY, NULL);
+    xTaskCreate(set_variable, (signed char *)"set_c", 128, (void *)&c, tskIDLE_PRIORITY, NULL);
+
+    TEST_ASSERT_FALSE_MESSAGE(a, "task set_a shouldn't run yet");
+    TEST_ASSERT_FALSE_MESSAGE(b, "task set_b shouldn't run yet");
+    TEST_ASSERT_FALSE_MESSAGE(c, "task set_c shouldn't run yet");
+
+    vTaskDelay(5);
+
+    TEST_ASSERT_TRUE_MESSAGE(a, "task set_a should have run");
+    TEST_ASSERT_TRUE_MESSAGE(b, "task set_b should have run");
+    TEST_ASSERT_TRUE_MESSAGE(c, "task set_c should have run");
+    TEST_PASS();
+}
+
+/* Verify that a high-priority task will starve a lower priority task */
+static void a_01_scheduler_priorities()
+{
+    /* Increase priority of the init task to make sure it always takes priority */
+    vTaskPrioritySet(xTaskGetCurrentTaskHandle(), tskIDLE_PRIORITY+4);
+
+    bool lower = false, higher = false;
+    xTaskHandle task_lower, task_higher;
+
+    xTaskCreate(set_variable, (signed char *)"high_prio", 128, (void *)&higher, tskIDLE_PRIORITY+1, &task_higher);
+    xTaskCreate(set_variable, (signed char *)"low_prio", 128, (void *)&lower, tskIDLE_PRIORITY, &task_lower);
+
+    TEST_ASSERT_FALSE_MESSAGE(higher, "higher prio task should not have run yet");
+    TEST_ASSERT_FALSE_MESSAGE(lower, "lower prio task should not have run yet");
+
+    vTaskDelay(2);
+
+    TEST_ASSERT_TRUE_MESSAGE(higher, "higher prio task should have run");
+    TEST_ASSERT_FALSE_MESSAGE(lower, "lower prio task should not have run");
+
+    /* Bump lower priority task over higher priority task */
+    vTaskPrioritySet(task_lower, tskIDLE_PRIORITY+2);
+
+    TEST_ASSERT_FALSE_MESSAGE(lower, "lower prio task should still not have run yet");
+
+    vTaskDelay(1);
+
+    TEST_ASSERT_TRUE_MESSAGE(lower, "lower prio task should have run");
+    TEST_PASS();
+}

tests/cases/02_heap.c

@@ -0,0 +1,56 @@
+#include "testcase.h"
+#include <malloc.h>
+#include <string.h>
+#include <FreeRTOS.h>
+
+DEFINE_SOLO_TESTCASE(02_heap_simple)
+DEFINE_SOLO_TESTCASE(02_heap_full)
+
+/* Simple heap accounting tests */
+static void a_02_heap_simple()
+{
+    struct mallinfo info = mallinfo();
+    printf("'arena' allocation size %d bytes\n", info.arena);
+    /* This is really a sanity check, if the "arena" size shrinks then
+       this is a good thing and we can update the test. If it grows
+       then we can also update the test, but we need a good reason. */
+    TEST_ASSERT_INT_WITHIN_MESSAGE(1000, 15000, info.arena, "Initial allocated heap should be approximately 15kB. SEE COMMENT.");
+
+    uint32_t freeheap = xPortGetFreeHeapSize();
+    printf("xPortGetFreeHeapSize = %d bytes\n", freeheap);
+    TEST_ASSERT_TRUE_MESSAGE(freeheap > 20000, "Should be at least 20kB free.");
+
+    uint8_t *buf = malloc(8192);
+    /* <-- have to do something with buf or gcc helpfully optimises it out! */
+    memset(buf, 0xEE, 8192);
+    uint32_t after = xPortGetFreeHeapSize();
+    struct mallinfo after_info = mallinfo();
+    printf("after arena size = %d bytes\n", after_info.arena);
+    printf("after xPortGetFreeHeapSize = %d bytes\n", after);
+    TEST_ASSERT_UINT32_WITHIN_MESSAGE(100, info.arena+8192, after_info.arena, "Allocated heap 'after' size should be 8kB more than before");
+    TEST_ASSERT_UINT32_WITHIN_MESSAGE(100, freeheap-8192, after, "Free heap size should be 8kB less than before");
+
+    free(buf);
+    after = xPortGetFreeHeapSize();
+    printf("after freeing xPortGetFreeHeapSize = %d bytes\n", after);
+    TEST_ASSERT_UINT32_WITHIN_MESSAGE(100, freeheap, after, "Free heap size after freeing buffer should be close to initial");
+    TEST_PASS();
+}
+
+/* Ensure malloc behaves when out of memory */
+static void a_02_heap_full()
+{
+    void *x = malloc(65536);
+    TEST_ASSERT_NULL_MESSAGE(x, "Allocating 64kB should fail and return null");
+
+    void *y = malloc(32768);
+    TEST_ASSERT_NOT_NULL_MESSAGE(y, "Allocating 32kB should succeed");
+
+    void *z = malloc(32768);
+    TEST_ASSERT_NULL_MESSAGE(z, "Allocating second 32kB should fail");
+
+    free(y);
+    z = malloc(32768);
+    TEST_ASSERT_NOT_NULL_MESSAGE(z, "Allocating 32kB should succeed after first block freed");
+    TEST_PASS();
+}

tests/cases/03_byte_load_flash.c

@@ -1,9 +1,14 @@
-/* Very basic example that just demonstrates we can run at all!
- */
+/* Unit tests to verify the "unaligned load handler" in core/exception_vectors.S that allows us to
+   complete byte loads from unaligned memory, etc.
+
+   Adapted from a test program in 'experiments' that did this.
+*/
+#include "testcase.h"
 #include "esp/rom.h"
 #include "esp/timer.h"
-#include "espressif/esp_common.h"
 #include "esp/uart.h"
+#include "espressif/esp_common.h"
+#include "xtensa_ops.h"
 #include "FreeRTOS.h"
 #include "task.h"
 #include "queue.h"
@@ -11,61 +16,97 @@
 #include "string.h"
 #include "strings.h"
 
+#include <malloc.h>
+
 #define TESTSTRING "O hai there! %d %d %d"
 
-const char *dramtest = TESTSTRING;
-const __attribute__((section(".iram1.notrodata"))) char iramtest[] = TESTSTRING;
-const __attribute__((section(".text.notrodata"))) char iromtest[] = TESTSTRING;
+static const char *dramtest = TESTSTRING;
+static const __attribute__((section(".iram1.notrodata"))) char iramtest[] = TESTSTRING;
+static const __attribute__((section(".text.notrodata"))) char iromtest[] = TESTSTRING;
+static const volatile __attribute__((section(".iram1.notliterals"))) int16_t unsigned_shorts[] = { -3, -4, -5, -32767, 44 };
+static const __attribute__((section(".iram1.notrodata"))) char sanity_test_data[] = {
+    0x01, 0x55, 0x7e, 0x2a, 0x81, 0xd5, 0xfe, 0xaa
+};
 
-static inline uint32_t get_ccount (void)
+DEFINE_SOLO_TESTCASE(03_byte_load_verify_sections)
+
+#define PTR_IN_REGION(PTR, START, LEN) ((START <= (intptr_t)(PTR)) && ((intptr_t)(PTR) < (START+LEN)))
+
+/* Sanity check, ensure the addresses of the various test strings are in the correct address space regions. */
+static void a_03_byte_load_verify_sections()
 {
-    uint32_t ccount;
-    asm volatile ("rsr.ccount %0" : "=a" (ccount));
-    return ccount;
+  printf("dramtest addr %p\n", dramtest);
+  TEST_ASSERT_MESSAGE(PTR_IN_REGION(dramtest, 0x3FFE8000, 0x14000), "dramtest should be in DRAM region");
+
+  printf("iramtest addr %p\n", iramtest);
+  TEST_ASSERT_MESSAGE(PTR_IN_REGION(iramtest, 0x40100000, 0x8000), "iramtest should be in IRAM region");
+
+  printf("iromtest addr %p\n", iromtest);
+  TEST_ASSERT_MESSAGE(PTR_IN_REGION(iromtest, 0x40220000, 0x100000), "iromtest sohuld be in IROM region");
+
+  printf("unsigned_shorts addr %p\n", unsigned_shorts);
+  TEST_ASSERT_MESSAGE(PTR_IN_REGION(unsigned_shorts, 0x40100000, 0x8000), "unsigned_shorts should be in IRAM region");
+
+  printf("sanity_test_data addr %p\n", sanity_test_data);
+  TEST_ASSERT_MESSAGE(PTR_IN_REGION(sanity_test_data, 0x40100000, 0x8000), "sanity_test_data should be in IRAM region");
+
+  TEST_PASS();
 }
 
-typedef void (* test_with_fn_t)(const char *string);
 
-char buf[64];
+/* test utility functions used for '03_byte_load_test_strings'
 
-void test_memcpy_aligned(const char *string)
+   returns the expected string result */
+typedef const char *(* test_with_fn_t)(const char *string);
+
+static char buf[64];
+
+static const char * test_memcpy_aligned(const char *string)
 {
     memcpy(buf, string, 16);
+    return "O hai there! %d ";
 }
 
-void test_memcpy_unaligned(const char *string)
+static const char * test_memcpy_unaligned(const char *string)
 {
     memcpy(buf, string, 15);
+    return "O hai there! %d";
 }
 
-void test_memcpy_unaligned2(const char *string)
+
+static const char * test_memcpy_unaligned2(const char *string)
 {
     memcpy(buf, string+1, 15);
+    return " hai there! %d ";
 }
 
-void test_strcpy(const char *string)
+static const char * test_strcpy(const char *string)
 {
     strcpy(buf, string);
+    return dramtest;
 }
 
-void test_sprintf(const char *string)
+static const char * test_sprintf(const char *string)
 {
     sprintf(buf, string, 1, 2, 3);
+    return "O hai there! 1 2 3";
 }
 
-void test_sprintf_arg(const char *string)
+static const char * test_sprintf_arg(const char *string)
 {
     sprintf(buf, "%s", string);
+    return dramtest;
 }
 
-void test_naive_strcpy(const char *string)
+static const char * test_naive_strcpy(const char *string)
 {
     char *to = buf;
     while((*to++ = *string++))
         ;
+    return dramtest;
 }
 
-void test_naive_strcpy_a0(const char *string)
+static const char * test_naive_strcpy_a0(const char *string)
 {
     asm volatile (
 "            mov          a8, %0    \n"
@@ -76,9 +117,10 @@ void test_naive_strcpy_a0(const char *string)
 "            addi.n       a8, a8, 1 \n"
 "            bnez         a0, tns_loop%=\n"
         : : "r" (buf), "r" (string) : "a0", "a8", "a9");
+    return dramtest;
 }
 
-void test_naive_strcpy_a2(const char *string)
+static const char * test_naive_strcpy_a2(const char *string)
 {
     asm volatile (
 "            mov          a8, %0    \n"
@@ -89,9 +131,10 @@ void test_naive_strcpy_a2(const char *string)
 "            addi.n       a8, a8, 1 \n"
 "            bnez         a2, tns_loop%=\n"
         : : "r" (buf), "r" (string) : "a2", "a8", "a9");
+    return dramtest;
 }
 
-void test_naive_strcpy_a3(const char *string)
+static const char * test_naive_strcpy_a3(const char *string)
 {
     asm volatile (
 "            mov          a8, %0    \n"
@@ -102,9 +145,10 @@ void test_naive_strcpy_a3(const char *string)
 "            addi.n       a8, a8, 1 \n"
 "            bnez         a3, tns_loop%=\n"
         : : "r" (buf), "r" (string) : "a3", "a8", "a9");
+    return TESTSTRING;
 }
 
-void test_naive_strcpy_a4(const char *string)
+static const char * test_naive_strcpy_a4(const char *string)
 {
     asm volatile (
 "            mov          a8, %0    \n"
@@ -115,9 +159,10 @@ void test_naive_strcpy_a4(const char *string)
 "            addi.n       a8, a8, 1 \n"
 "            bnez         a4, tns_loop%=\n"
         : : "r" (buf), "r" (string) : "a4", "a8", "a9");
+    return TESTSTRING;
 }
 
-void test_naive_strcpy_a5(const char *string)
+static const char * test_naive_strcpy_a5(const char *string)
 {
     asm volatile (
 "            mov          a8, %0    \n"
@@ -128,9 +173,10 @@ void test_naive_strcpy_a5(const char *string)
 "            addi.n       a8, a8, 1 \n"
 "            bnez         a5, tns_loop%=\n"
         : : "r" (buf), "r" (string) : "a5", "a8", "a9");
+    return TESTSTRING;
 }
 
-void test_naive_strcpy_a6(const char *string)
+static const char * test_naive_strcpy_a6(const char *string)
 {
     asm volatile (
 "            mov          a8, %0    \n"
@@ -141,102 +187,79 @@ void test_naive_strcpy_a6(const char *string)
 "            addi.n       a8, a8, 1 \n"
 "            bnez         a6, tns_loop%=\n"
         : : "r" (buf), "r" (string) : "a6", "a8", "a9");
+    return TESTSTRING;
 }
 
-void test_l16si(const char *string)
+static const char * test_noop(const char *string)
 {
-    /* This follows most of the l16si path, but as the
-     values in the string are all 7 bit none of them get sign extended.
-
-    See separate test_sign_extension function which validates
-    sign extension works as expected.
-    */
-    int16_t *src_int16 = (int16_t *)string;
-    int32_t *dst_int32 = (int32_t *)buf;
-    dst_int32[0] = src_int16[0];
-    dst_int32[1] = src_int16[1];
-    dst_int32[2] = src_int16[2];
+  buf[0] = 0;
+  return "";
 }
 
-#define TEST_REPEATS 1000
-
-void test_noop(const char *string)
-{
-
-}
-
-uint32_t IRAM run_test(const char *string, test_with_fn_t testfn, const char *testfn_label, uint32_t nullvalue, bool evict_cache)
+static uint32_t IRAM inner_string_test(const char *string, test_with_fn_t testfn, const char *testfn_label, uint32_t nullvalue, bool evict_cache)
 {
     printf(" .. against %30s: ", testfn_label);
     vPortEnterCritical();
-    uint32_t before = get_ccount();
+    uint32_t before;
+    RSR(before, CCOUNT);
+    const int TEST_REPEATS = 1000;
     for(int i = 0; i < TEST_REPEATS; i++) {
-        testfn(string);
+      memset(buf, 0, sizeof(buf));
+      const char *expected = testfn(string);
+      TEST_ASSERT_EQUAL_STRING_MESSAGE(expected, buf, testfn_label);
       if(evict_cache) {
 	Cache_Read_Disable();
 	Cache_Read_Enable(0,0,1);
       }
     }
-    uint32_t after = get_ccount();
+    uint32_t after;
+    RSR(after, CCOUNT);
     vPortExitCritical();
     uint32_t instructions = (after-before)/TEST_REPEATS - nullvalue;
     printf("%5d instructions\r\n", instructions);
     return instructions;
 }
 
-void test_string(const char *string, char *label, bool evict_cache)
+static void string_test(const char *string, char *label, bool evict_cache)
 {
     printf("Testing %s (%p) '%s'\r\n", label, string, string);
     printf("Formats as: '");
     printf(string, 1, 2, 3);
     printf("'\r\n");
-    uint32_t nullvalue = run_test(string, test_noop, "null op", 0, evict_cache);
-    run_test(string, test_memcpy_aligned, "memcpy - aligned len", nullvalue, evict_cache);
-    run_test(string, test_memcpy_unaligned, "memcpy - unaligned len", nullvalue, evict_cache);
-    run_test(string, test_memcpy_unaligned2, "memcpy - unaligned start&len", nullvalue, evict_cache);
-    run_test(string, test_strcpy, "strcpy", nullvalue, evict_cache);
-    run_test(string, test_naive_strcpy, "naive strcpy", nullvalue, evict_cache);
-    run_test(string, test_naive_strcpy_a0, "naive strcpy (a0)", nullvalue, evict_cache);
-    run_test(string, test_naive_strcpy_a2, "naive strcpy (a2)", nullvalue, evict_cache);
-    run_test(string, test_naive_strcpy_a3, "naive strcpy (a3)", nullvalue, evict_cache);
-    run_test(string, test_naive_strcpy_a4, "naive strcpy (a4)", nullvalue, evict_cache);
-    run_test(string, test_naive_strcpy_a5, "naive strcpy (a5)", nullvalue, evict_cache);
-    run_test(string, test_naive_strcpy_a6, "naive strcpy (a6)", nullvalue, evict_cache);
-    run_test(string, test_sprintf, "sprintf", nullvalue, evict_cache);
-    run_test(string, test_sprintf_arg, "sprintf format arg", nullvalue, evict_cache);
-    run_test(string, test_l16si, "load as l16si", nullvalue, evict_cache);
+    uint32_t nullvalue = inner_string_test(string, test_noop, "null op", 0, evict_cache);
+    inner_string_test(string, test_memcpy_aligned, "memcpy - aligned len", nullvalue, evict_cache);
+    inner_string_test(string, test_memcpy_unaligned, "memcpy - unaligned len", nullvalue, evict_cache);
+    inner_string_test(string, test_memcpy_unaligned2, "memcpy - unaligned start&len", nullvalue, evict_cache);
+    inner_string_test(string, test_strcpy, "strcpy", nullvalue, evict_cache);
+    inner_string_test(string, test_naive_strcpy, "naive strcpy", nullvalue, evict_cache);
+    inner_string_test(string, test_naive_strcpy_a0, "naive strcpy (a0)", nullvalue, evict_cache);
+    inner_string_test(string, test_naive_strcpy_a2, "naive strcpy (a2)", nullvalue, evict_cache);
+    inner_string_test(string, test_naive_strcpy_a3, "naive strcpy (a3)", nullvalue, evict_cache);
+    inner_string_test(string, test_naive_strcpy_a4, "naive strcpy (a4)", nullvalue, evict_cache);
+    inner_string_test(string, test_naive_strcpy_a5, "naive strcpy (a5)", nullvalue, evict_cache);
+    inner_string_test(string, test_naive_strcpy_a6, "naive strcpy (a6)", nullvalue, evict_cache);
+    inner_string_test(string, test_sprintf, "sprintf", nullvalue, evict_cache);
+    inner_string_test(string, test_sprintf_arg, "sprintf format arg", nullvalue, evict_cache);
 }
 
-static void test_isr();
-static void test_sign_extension();
-static void test_system_interaction();
-void sanity_tests(void);
+DEFINE_SOLO_TESTCASE(03_byte_load_test_strings)
 
-void user_init(void)
+/* Test various operations on strings in various regions */
+static void a_03_byte_load_test_strings()
 {
-    uart_set_baud(0, 115200);
-
-    gpio_enable(2, GPIO_OUTPUT); /* used for LED debug */
-    gpio_write(2, 1); /* active low */
-
-    printf("\r\n\r\nSDK version:%s\r\n", sdk_system_get_sdk_version());
-    sanity_tests();
-    test_string(dramtest, "DRAM", 0);
-    test_string(iramtest, "IRAM", 0);
-    test_string(iromtest, "Cached flash", 0);
-    test_string(iromtest, "'Uncached' flash", 1);
-
-    test_isr();
-    test_sign_extension();
-
-    xTaskHandle taskHandle;
-    xTaskCreate(test_system_interaction, (signed char *)"interactionTask", 256, &taskHandle, 2, NULL);
+    string_test(dramtest, "DRAM", 0);
+    string_test(iramtest, "IRAM", 0);
+    string_test(iromtest, "Cached flash", 0);
+    string_test(iromtest, "'Uncached' flash", 1);
+    TEST_PASS();
 }
 
 static volatile bool frc1_ran;
 static volatile bool frc1_finished;
 static volatile char frc1_buf[80];
 
+DEFINE_SOLO_TESTCASE(03_byte_load_test_isr)
+
 static void frc1_interrupt_handler(void)
 {
     frc1_ran = true;
@@ -245,7 +268,8 @@ static void frc1_interrupt_handler(void)
     frc1_finished = true;
 }
 
-static void test_isr()
+/* Verify that the unaligned loader can run inside an ISR */
+static void a_03_byte_load_test_isr()
 {
     printf("Testing behaviour inside ISRs...\r\n");
     timer_set_interrupts(FRC1, false);
@@ -257,26 +281,27 @@ static void test_isr()
     sdk_os_delay_us(2000);
 
     if(!frc1_ran)
-        printf("ERROR: FRC1 timer exception never fired.\r\n");
+      TEST_FAIL_MESSAGE("ERROR: FRC1 timer exception never fired.\r\n");
     else if(!frc1_finished)
-        printf("ERROR: FRC1 timer exception never finished.\r\n");
+      TEST_FAIL_MESSAGE("ERROR: FRC1 timer exception never finished.\r\n");
     else if(strcmp((char *)frc1_buf, iramtest))
-        printf("ERROR: FRC1 strcpy from IRAM failed.\r\n");
+      TEST_FAIL_MESSAGE("ERROR: FRC1 strcpy from IRAM failed.\r\n");
     else
-        printf("PASSED\r\n");
+      TEST_PASS();
 }
 
-const volatile __attribute__((section(".iram1.notliterals"))) int16_t unsigned_shorts[] = { -3, -4, -5, -32767, 44 };
+DEFINE_SOLO_TESTCASE(03_byte_load_test_sign_extension)
 
-static void test_sign_extension()
+static void a_03_byte_load_test_sign_extension()
 {
     /* this step seems to be necessary so the compiler will actually generate l16si */
     int16_t *shorts_p = (int16_t *)unsigned_shorts;
     if(shorts_p[0] == -3 && shorts_p[1] == -4 && shorts_p[2] == -5 && shorts_p[3] == -32767 && shorts_p[4] == 44)
     {
-        printf("l16si sign extension PASSED.\r\n");
+      TEST_PASS();
     } else {
-        printf("ERROR: l16si sign extension failed. Got values %d %d %d %d %d\r\n", shorts_p[0], shorts_p[1], shorts_p[2], shorts_p[3], shorts_p[4]);
+      sprintf(buf, "l16si sign extension failed. Got values %d %d %d %d %d\r\n", shorts_p[0], shorts_p[1], shorts_p[2], shorts_p[3], shorts_p[4]);
+      TEST_FAIL_MESSAGE(buf);
     }
 }
 
@@ -285,11 +310,13 @@ static void test_sign_extension()
 
    The following tests run inside a FreeRTOS task, after everything else.
 */
-static void test_system_interaction()
+DEFINE_SOLO_TESTCASE(03_byte_load_test_system_interaction);
+
+static void task_load_test_system_interaction()
 {
     uint32_t start = xTaskGetTickCount();
-    printf("Starting system/timer interaction test (takes approx 30 seconds)...\n");
-    for(int i = 0; i < 200*1000; i++) {
+    printf("Starting system/timer interaction test (takes approx 1 second)...\n");
+    for(int i = 0; i < 5000; i++) {
         test_naive_strcpy_a0(iromtest);
         test_naive_strcpy_a2(iromtest);
         test_naive_strcpy_a3(iromtest);
@@ -304,13 +331,22 @@ static void test_system_interaction()
         */
     }
     uint32_t ticks = xTaskGetTickCount() - start;
-    printf("Timer interaction test PASSED after %dms.\n", ticks*portTICK_RATE_MS);
-    while(1) {}
+    printf("Timer interaction test PASSED after %d ticks.\n", ticks);
+    TEST_PASS();
+}
+
+static void a_03_byte_load_test_system_interaction()
+{
+    xTaskCreate(task_load_test_system_interaction, (signed char *)"interactionTask", 256, NULL, 2, NULL);
+    while(1) {
+      vTaskDelay(100);
+    }
 }
 
 /* The following "sanity tests" are designed to try to execute every code path
  * of the LoadStoreError handler, with a variety of offsets and data values
  * designed to catch any mask/shift errors, sign-extension bugs, etc */
+DEFINE_SOLO_TESTCASE(03_byte_load_test_sanity)
 
 /* (Contrary to expectations, 'mov a15, a15' in Xtensa is not technically a
  * no-op, but is officially "undefined and reserved for future use", so we need
@@ -339,17 +375,13 @@ static void test_system_interaction()
         if (result != value) sanity_test_failed(op, reg, addr, value, result); \
     }
 
-void sanity_test_failed(const char *testname, const char *reg, const void *addr, int32_t value, int32_t result) {
+static void sanity_test_failed(const char *testname, const char *reg, const void *addr, int32_t value, int32_t result) {
     uint32_t actual_data = *(uint32_t *)((uint32_t)addr & 0xfffffffc);
-
-    printf("*** SANITY TEST FAILED: '%s %s' from %p (underlying 32-bit value: 0x%x): Expected 0x%08x (%d), got 0x%08x (%d)\n", testname, reg, addr, actual_data, value, value, result, result);
+    sprintf(buf, "%s %s from %p (32-bit value: 0x%x): Expected 0x%08x (%d), got 0x%08x (%d)\n", testname, reg, addr, actual_data, value, value, result, result);
+    TEST_FAIL_MESSAGE(buf);
 }
 
-const __attribute__((section(".iram1.notrodata"))) char sanity_test_data[] = {
-    0x01, 0x55, 0x7e, 0x2a, 0x81, 0xd5, 0xfe, 0xaa
-};
-
-void sanity_test_l8ui(const void *addr, int32_t value) {
+static void sanity_test_l8ui(const void *addr, int32_t value) {
     TEST_LOAD("l8ui", "a0", addr, value);
     TEST_LOAD("l8ui", "a1", addr, value);
     TEST_LOAD("l8ui", "a2", addr, value);
@@ -368,7 +400,7 @@ void sanity_test_l8ui(const void *addr, int32_t value) {
     TEST_LOAD("l8ui", "a15", addr, value);
 }
 
-void sanity_test_l16ui(const void *addr, int32_t value) {
+static void sanity_test_l16ui(const void *addr, int32_t value) {
     TEST_LOAD("l16ui", "a0", addr, value);
     TEST_LOAD("l16ui", "a1", addr, value);
     TEST_LOAD("l16ui", "a2", addr, value);
@@ -387,7 +419,7 @@ void sanity_test_l16ui(const void *addr, int32_t value) {
     TEST_LOAD("l16ui", "a15", addr, value);
 }
 
-void sanity_test_l16si(const void *addr, int32_t value) {
+static void sanity_test_l16si(const void *addr, int32_t value) {
     TEST_LOAD("l16si", "a0", addr, value);
     TEST_LOAD("l16si", "a1", addr, value);
     TEST_LOAD("l16si", "a2", addr, value);
@@ -406,7 +438,7 @@ void sanity_test_l16si(const void *addr, int32_t value) {
     TEST_LOAD("l16si", "a15", addr, value);
 }
 
-void sanity_tests(void) {
+static void a_03_byte_load_test_sanity(void) {
     printf("== Performing sanity tests (sanity_test_data @ %p)...\n", sanity_test_data);
 
     sanity_test_l8ui(sanity_test_data + 0, 0x01);
@@ -429,4 +461,5 @@ void sanity_tests(void) {
     sanity_test_l16si(sanity_test_data + 6, -21762);
 
     printf("== Sanity tests completed.\n");
+    TEST_PASS();
 }

tests/include/testcase.h

@@ -0,0 +1,58 @@
+#ifndef _TESTCASE_H
+#define _TESTCASE_H
+#include <stdbool.h>
+#include <stdio.h>
+#include "esp/uart.h"
+
+/* Unity is the framework with test assertions, etc. */
+#include "unity.h"
+
+/* Need to explicitly flag once a test has completed successfully. */
+#define TEST_PASS() do { UnityConcludeTest(); while(1) { } } while (0)
+
+/* Types of test, defined by hardware requirements */
+typedef enum {
+    SOLO,        /* Test require "ESP A" only, no other connections */
+    DUAL,        /* Test requires "ESP A" and "ESP "B", basic interconnections between them */
+    EYORE_TEST,  /* Test requires an eyore-test board with onboard STM32F0 */
+} testcase_type_t;
+
+typedef void (testcase_fn_t)(void);
+
+typedef struct {
+    const char *name;
+    const char *file;
+    int line;
+    testcase_type_t type;
+    testcase_fn_t *a_fn;
+    testcase_fn_t *b_fn;
+} testcase_t;
+
+void testcase_register(const testcase_t *testcase);
+
+/* Register a test case using these macros. Use DEFINE_SOLO_TESTCASE for single-MCU tests,
+   and DEFINE_TESTCASE for all other test types.
+*/
+#define DEFINE_SOLO_TESTCASE(NAME)                                      \
+    static testcase_fn_t a_##NAME;                                      \
+    _DEFINE_TESTCASE_COMMON(NAME, SOLO, a_##NAME, 0)
+
+#define DEFINE_TESTCASE(NAME, TYPE)                                     \
+    static testcase_fn_t a_##NAME;                                      \
+    static testcase_fn_t b_##NAME;                                      \
+    _DEFINE_TESTCASE_COMMON(NAME, TYPE, A_##NAME, B_##NAME)
+
+
+#define _DEFINE_TESTCASE_COMMON(NAME, TYPE, A_FN, B_FN)                 \
+    void __attribute__((constructor)) testcase_ctor_##NAME() {          \
+        const testcase_t testcase = { .name = #NAME,                    \
+                                      .file = __FILE__,                 \
+                                      .line = __LINE__,                 \
+                                      .type = TYPE,                     \
+                                      .a_fn = A_FN,                     \
+                                      .b_fn = B_FN,                     \
+        };                                                              \
+        testcase_register(&testcase);                                   \
+    }
+
+#endif

tests/test_main.c

@@ -0,0 +1,115 @@
+#include "testcase.h"
+#include <stdlib.h>
+#include <esp/uart.h>
+#include <string.h>
+
+/* Convert requirement enum to a string we can print */
+static const char *get_requirements_name(const testcase_type_t arg) {
+    switch(arg) {
+    case SOLO:
+        return "SOLO";
+    case DUAL:
+        return "DUAL";
+    case EYORE_TEST:
+        return "EYORE_TEST";
+    default:
+        return "UNKNOWN";
+    }
+}
+
+static testcase_t *testcases;
+static uint32_t testcases_count;
+static uint32_t testcases_alloc;
+
+void testcase_register(const testcase_t *testcase)
+{
+    /* Grow the testcases buffer to fit the new test case,
+       this buffer will be freed before the test runs.
+     */
+    if(testcases_count == testcases_alloc) {
+        testcases_alloc += 1;
+        testcases = realloc(testcases, testcases_alloc * sizeof(testcase_t));
+        if(!testcases) {
+            printf("Failed to reallocate test case register length %d\n", testcases_alloc);
+            testcases_count = 0;
+            testcases_alloc = 0;
+        }
+    }
+    memcpy(&testcases[testcases_count++], testcase, sizeof(testcase_t));
+}
+
+void user_init(void)
+{
+    uart_set_baud(0, 115200);
+    printf("esp-open-rtos test runner.\n");
+    printf("%d test cases are defined:\n\n", testcases_count);
+    for(int i = 0; i < testcases_count; i++) {
+        printf("CASE %d = %s %s\n", i, testcases[i].name, get_requirements_name(testcases[i].type));
+    }
+
+    printf("Enter A or B then number of test case to run, ie A0.\n");
+    int case_idx = -1;
+    char type;
+    do {
+        printf("> ");
+        uart_rxfifo_wait(0,1);
+        type = uart_getc(0);
+        if(type != 'a' && type != 'A' && type != 'b' && type != 'B') {
+            printf("Type must be A or B.\n");
+            continue;
+        }
+
+        char idx_buf[6];
+        for(int c = 0; c < sizeof(idx_buf); c++) {
+            uart_rxfifo_wait(0,1);
+            idx_buf[c] = uart_getc(0);
+            if(idx_buf[c] == ' ') { /* Ignore spaces */
+                c--;
+                continue;
+            }
+            if(idx_buf[c] == '\r' || idx_buf[c] == '\n') {
+                idx_buf[c] = 0;
+                case_idx = atoi(idx_buf);
+                break;
+            }
+            else if(idx_buf[c] < '0' || idx_buf[c] > '9') {
+                break;
+            }
+        }
+
+        if(case_idx == -1) {
+            printf("Invalid case index");
+        }
+        else if(case_idx < 0 || case_idx >= testcases_count) {
+            printf("Test case index out of range.\n");
+        }
+        else if((type == 'b' || type =='B') && testcases[case_idx].type == SOLO) {
+            printf("No ESP type B for 'SOLO' test cases.\n");
+        } else {
+            break;
+        }
+    } while(1);
+    if(type =='a')
+        type = 'A';
+    else if (type == 'b')
+        type = 'B';
+    testcase_t testcase;
+    memcpy(&testcase, &testcases[case_idx], sizeof(testcase_t));
+    /* Free the register of test cases now we have the one we're running */
+    free(testcases);
+    testcases_alloc = 0;
+    testcases_count = 0;
+
+    printf("\nRunning test case %d (%s %s) as instance %c \nDefinition at %s:%d\n***\n", case_idx,
+           testcase.name, get_requirements_name(testcase.type), type,
+           testcase.file, testcase.line);
+    Unity.CurrentTestName = testcase.name;
+    Unity.TestFile = testcase.file;
+    Unity.CurrentTestLineNumber = testcase.line;
+    Unity.NumberOfTests = 1;
+    if(type=='A')
+        testcase.a_fn();
+    else
+        testcase.b_fn();
+    TEST_FAIL_MESSAGE("\n\nTest initialisation routine returned without calling TEST_PASS. Buggy test?");
+}

tests/test_runner.py

@@ -0,0 +1,339 @@
+#!/usr/bin/env python3
+import sys
+import argparse
+import subprocess
+import os
+import serial
+import threading
+import re
+import time
+import traceback
+
+SHORT_OUTPUT_TIMEOUT=0.25 # timeout for resetting and/or waiting for more lines of output
+TESTCASE_TIMEOUT=10
+TESTRUNNER_BANNER="esp-open-rtos test runner."
+
+def main():
+    global verbose
+    args = parse_args()
+    verbose = args.verbose
+
+    if not args.no_flash:
+        flash_image(args.aport)
+        if args.type != 'solo':
+            flash_image(args.bport)
+
+    env = TestEnvironment(args.aport, TestEnvironment.A)
+    cases = env.get_testlist()
+    if args.type != 'solo':
+        env_b = TestEnvironment(args.bport, TestEnvironment.B)
+        cases_b = env_b.get_testlist()
+        if cases != cases_b:
+            raise TestRunnerError("Test cases on units A & B don't match")
+
+    counts = dict((status,0) for status in TestResult.STATUS_NAMES.keys())
+    failures = False
+    for test in cases:
+        res = test.run(env)
+        counts[res.status] += 1
+        failures = failures or res.is_failure()
+
+    print("%20s: %d" % ("Total tests", sum(c for c in counts.values())))
+    print()
+    # print status counts for tests
+    for c in sorted(counts.keys()):
+        print("%20s: %d" % (TestResult.STATUS_NAMES[c], counts[c]))
+
+    sys.exit(1 if failures else 0)
+
+class TestCase(object):
+    def __init__(self, index, name, case_type):
+        self.name = name
+        self.index = index
+        self.case_type = case_type
+
+    def __repr__(self):
+        return "#%d: %s (%s)" % (self.index, self.name, self.case_type)
+
+    def __eq__(self, other):
+        return (self.index == other.index
+                and self.name == other.name
+                and self.case_type == other.case_type)
+
+    def run(self, env_a, env_b = None):
+        """
+        Run the test represented by this instance, against the environment(s) passed in.
+
+        Returns a TestResult
+        """
+        sys.stdout.write("Running test case '%s'...%s" % (self.name, "\n" if verbose else " "*(40-len(self.name))))
+        mon_a = env_a.start_testcase(self)
+        mon_b = env_b.start_testcase(self) if env_b else None
+        while True:
+            if mon_a.get_result() and (mon_b is None or mon_b.get_result()):
+                break # all running test environments have finished
+
+            # or, in the case both are running, stop as soon as either environemnt shows a failure
+            try:
+                if mon_a.get_result().is_failure():
+                    mon_b.cancel()
+                    break
+            except AttributeError:
+                pass
+            try:
+                if mon_b.get_result().is_failure():
+                    mon_a.cancel()
+                    break
+            except AttributeError:
+                pass
+            time.sleep(0.1)
+
+        if mon_b is not None:
+            # return whichever result is more severe
+            res = max(mon_a.get_result(), mon_b.get_result())
+        else:
+            res = mon_a.get_result()
+        if not verbose: # finish the line after the ...
+            print(TestResult.STATUS_NAMES[res.status])
+            if res.is_failure():
+                message = res.message
+                if "/" in res.message: # cut anything before the file name in the failure
+                    message = message[message.index("/"):]
+                print("FAILURE MESSAGE:\n%s\n" % message)
+        return res
+
+class TestResult(object):
+    """ Class to wrap a test result code and a message """
+    # Test status flags, higher = more severe
+    CANCELLED = 0
+    SKIPPED = 1
+    PASSED = 2
+    FAILED = 3
+    ERROR = 4
+
+    STATUS_NAMES = {
+        CANCELLED : "Cancelled",
+        SKIPPED : "Skipped",
+        PASSED : "Passed",
+        FAILED : "Failed",
+        ERROR : "Error"
+        }
+
+    def __init__(self, status, message):
+        self.status = status
+        self.message = message
+
+    def is_failure(self):
+        return self.status >= TestResult.FAILED
+
+    def __cmp__(self, other):
+        if other is None:
+            return 1
+        return self.status - other.status
+
+class TestMonitor(object):
+    """ Class to monitor a running test case in a separate thread, defer reporting of the result until it's done.
+
+    Can poll for completion by calling is_done(), read a TestResult via .get_result()
+    """
+    def __init__(self, port, instance):
+        super(TestMonitor, self).__init__()
+        self._thread = threading.Thread(target=self._monitorThread)
+        self._port = port
+        self._instance = instance
+        self._result = None
+        self._cancelled = False
+        self.output = ""
+        self._thread.start()
+
+    def cancel(self):
+        self._cancelled = True
+
+    def is_done(self):
+        return self._result is not None
+
+    def get_result(self):
+        return self._result
+
+    def _monitorThread(self):
+        self.output = ""
+        start_time = time.time()
+        self._port.timeout = SHORT_OUTPUT_TIMEOUT
+        try:
+            while not self._cancelled and time.time() < start_time + TESTCASE_TIMEOUT:
+                line = self._port.readline().decode("utf-8", "ignore")
+                if line == "":
+                    continue # timed out
+                self.output += "%s+%4.2fs %s" % (self._instance, time.time()-start_time, line)
+                verbose_print(line.strip())
+                if line.endswith(":PASS\r\n"):
+                    self._result = TestResult(TestResult.PASSED, "Test passed.")
+                    return
+                elif ":FAIL:" in line:
+                    self._result = TestResult(TestResult.FAILED, line)
+                    return
+                elif line == TESTRUNNER_BANNER:
+                    self._result = TestResult(TestResult.ERROR, "Test caused crash and reset.")
+                    return
+            if not self._cancelled:
+                self._result = TestResult(TestResult.CANCELLED, "Cancelled")
+            else:
+                self._result = TestResult(TestResult.ERROR, "Test timed out")
+
+        finally:
+            self._port.timeout = None
+
+class TestEnvironment(object):
+    A = "A"
+    B = "B"
+
+    def __init__(self, port, instance):
+        self._name = port
+        self._port = TestSerialPort(port, baudrate=115200)
+        self._instance = instance
+
+    def reset(self):
+        """ Resets the test board, and waits for the test runner program to start up """
+        self._port.setDTR(False)
+        self._port.setRTS(True)
+        time.sleep(0.05)
+        self._port.flushInput()
+        self._port.setRTS(False)
+        verbose_print("Waiting for test runner startup...")
+        if not self._port.wait_line(lambda line: line == TESTRUNNER_BANNER):
+            raise TestRunnerError("Port %s failed to start test runner" % self._port)
+
+    def get_testlist(self):
+        """ Resets the test board and returns the enumerated list of all supported tests """
+        self.reset()
+        tests = []
+        verbose_print("Enumerating tests...")
+
+        def collect_testcases(line):
+                if line.startswith(">"):
+                    return True # prompt means list of test cases is done, success
+                m = re.match(r"CASE (\d+) = (.+?) ([A-Z]+)", line)
+                if m is not None:
+                    t = TestCase(int(m.group(1)), m.group(2), m.group(3).lower())
+                    verbose_print(t)
+                    tests.append(t)
+        if not self._port.wait_line(collect_testcases):
+            raise TestRunnerError("Port %s failed to read test list" % self._port)
+        verbose_print("Port %s found %d test cases" % (self._name, len(tests)))
+        return tests
+
+    def start_testcase(self, case):
+        """ Starts the specified test instance and returns an TestMonitor reader thread instance to monitor the output """
+        # synchronously start the test case
+        self.reset()
+        if not self._port.wait_line(lambda line: line.startswith(">")):
+            raise TestRunnerError("Failed to read test runnner prompt")
+        command = "%s%d\r\n" % (self._instance, case.index)
+        self._port.write(command.encode("utf-8"))
+        return TestMonitor(self._port, self._instance)
+
+
+def get_testdir():
+    """
+    Return the 'tests' directory in the source tree
+    (assuming the test_runner.py script is in that directory.
+    """
+    res = os.path.dirname(__name__)
+    return "." if res == "" else res
+
+
+def flash_image(serial_port):
+    # Bit hacky: rather than calling esptool directly, just use the Makefile flash target
+    # with the correct ESPPORT argument
+    env = dict(os.environ)
+    env["ESPPORT"] = serial_port
+    verbose_print("Building and flashing test image to %s..." % serial_port)
+    try:
+        stdout = sys.stdout if verbose else None
+        output = subprocess.run(["make","flash"], check=True, cwd=get_testdir(), stdout=stdout, stderr=subprocess.STDOUT, env=env)
+    except subprocess.CalledProcessError as e:
+        raise TestRunnerError("'make flash EPPORT=%s' failed with exit code %d" % (serial_port, e.returncode))
+    verbose_print("Flashing successful.")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='esp-open-rtos testrunner', prog='test_runner')
+
+    parser.add_argument(
+        '--type', '-t',
+        help='Type of test hardware attached to serial ports A & (optionally) B',
+        choices=['solo','dual','eyore_test'], default='solo')
+
+    parser.add_argument(
+        '--aport', '-a',
+        help='Serial port for device A',
+        default='/dev/ttyUSB0')
+
+    parser.add_argument(
+        '--bport', '-b',
+        help='Serial port for device B (ignored if type is \'solo\')',
+        default='/dev/ttyUSB1')
+
+    parser.add_argument(
+        '--no-flash', '-n',
+        help='Don\'t flash the test binary image before running tests',
+        action='store_true',
+        default=False)
+
+    parser.add_argument(
+        '--verbose', '-v',
+        help='Verbose test runner debugging output',
+        action='store_true',
+        default=False)
+
+    parser.add_argument('testcases', nargs='*',
+                        help='Optional list of test cases to run. By default, all tests are run.')
+
+    return parser.parse_args()
+
+
+class TestRunnerError(RuntimeError):
+    def __init__(self, message):
+        RuntimeError.__init__(self, message)
+
+class TestSerialPort(serial.Serial):
+    def __init__(self, *args, **kwargs):
+        super(TestSerialPort, self).__init__(*args, **kwargs)
+
+    def wait_line(self, callback, timeout = SHORT_OUTPUT_TIMEOUT):
+        """ Wait for the port to output a particular piece of line content, as judged by callback
+
+            Callback called as 'callback(line)' and returns not-True if non-match otherwise can return any value.
+
+            Returns first non-False result from the callback, or None if it timed out waiting for a new line.
+
+            Note that a serial port spewing legitimate lines of output may block this function forever, if callback
+            doesn't detect this is happening.
+        """
+        self.timeout = timeout
+        try:
+            res = None
+            while not res:
+                line = self.readline()
+                if line == b"":
+                    break # timed out
+                line = line.decode("utf-8", "ignore").rstrip()
+                res = callback(line)
+            return res
+        finally:
+            self.timeout = None
+
+verbose = False
+
+
+def verbose_print(msg):
+    if verbose:
+        print(msg)
+
+if __name__ == '__main__':
+    try:
+        main()
+    except TestRunnerError as e:
+        print(e)
+        sys.exit(2)
+

tests/unity

@@ -0,0 +1 @@
+Subproject commit 7943c766b993c9a84e1f6661d2d2427f6f2df9d0