Draft version of test system

.gitmodules

@@ -13,6 +13,6 @@
 [submodule "extras/spiffs/spiffs"]
 	path = extras/spiffs/spiffs
 	url = https://github.com/pellepl/spiffs.git
-[submodule "examples/posix_fs/fs-test"]
-	path = examples/posix_fs/fs-test
+[submodule "tests/spiffs/fs-test"]
+	path = tests/spiffs/fs-test
 	url = https://github.com/sheinz/fs-test

extras/test/component.mk

@@ -0,0 +1,8 @@
+# Component makefile for extras/test
+
+INC_DIRS += $(test_ROOT)..
+
+# args for passing into compile rule generation
+test_SRC_DIR =  $(test_ROOT)
+
+$(eval $(call component_compile_rules,test))

extras/test/run_test.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This script reads specified serial port for test output. If the test is
+# successfully finished the script returns 0 otherwise -1. The script also
+# checks for time-out. The timeout value is received from the test itself
+# during test initialization.
+#
+# The script runs as many instances as many serial devices provided.
+#
+# Usage:
+# > run_test.py /dev/ttyUSB0
+# or
+# > run_test.py /dev/ttyUSB0 /dev/ttyUSB1 /dev/ttyUSB2
+# if three devices are connected and should be tested simultaneously
+#
+
+
+import serial
+import re
+import sys
+import time
+import threading
+
+
+class Test(threading.Thread):
+    def __init__(self, port, baudrate):
+        threading.Thread.__init__(self)
+        print("init: {}".format(port))
+        self._port = serial.Serial(port, baudrate, timeout=1)
+        self._timeout = None
+        self.result = None
+
+    def wait_start(self):
+        skip_lines = 100
+        while skip_lines > 0:
+            l = self._port.readline()
+            if 'TEST_INIT: TIMEOUT=' in l:
+                res = re.findall(r'\d+', l)
+                if len(res) != 1:
+                    print("Invalid timeout format")
+                    self.result = False
+                    return
+                else:
+                    self._timeout = int(res[0])
+                break
+            else:
+                sys.stdout.write(l)
+                skip_lines -= 1
+        if self._timeout is None:
+            print("Device is not responding")
+            self.result = False
+        else:
+            print("Timeout: {}".format(self._timeout))
+            self.result = True
+
+    def test(self):
+        end_time = time.time() + self._timeout
+        result = None
+        while end_time > time.time():
+            l = self._port.readline()
+            if 'TEST_FINISH: ' in l:
+                if 'SUCCESS' in l:
+                    print("Test finished successfully")
+                    result = True
+                    break
+                elif 'FAIL' in l:
+                    print("Test failed")
+                    result = False
+                else:
+                    print("Unknown status")
+                    result = False
+            else:
+                if l is not None:
+                    sys.stdout.write(l)
+        if result is None:
+            print("Test timeout")
+        return result
+
+    def run(self):
+        self.wait_start()
+        if self.result is not None:
+            self.result = self.test()
+
+    def __del__(self):
+        self._port.close()
+
+
+def main():
+    if len(sys.argv) <= 1:
+        print("Need at least one serial device to test a device")
+        exit(-1)
+
+    tasks = []
+    for dev in sys.argv[1:]:
+        t = Test(dev, 115200)
+        t.start()
+        tasks.append(t)
+
+    result = True
+    for t in tasks:
+        t.join()
+        result = result and t.result
+
+    if result:
+        print("Test: SUCCESS")
+    else:
+        print("Test: FAIL")
+
+if __name__ == "__main__":
+    main()

extras/test/test.c

@@ -0,0 +1,27 @@
+/**
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 sheinz (https://github.com/sheinz)
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "test.h"
+
+bool test_result = true;

extras/test/test.h

@@ -0,0 +1,67 @@
+/**
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 sheinz (https://github.com/sheinz)
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef __TEST_H__
+#define __TEST_H__
+
+#include <stdio.h>
+#include <stdbool.h>
+
+#include "esp/uart.h"
+#include "FreeRTOS.h"
+
+extern bool test_result;
+
+#define TEST_CHECK(x, msg)  \
+    if (!(x)) { \
+        printf("FAIL:%s:%d:%s\n", __FILE__, __LINE__, (msg)); \
+        test_result = false; \
+    }
+
+#define TEST_ASSERT(x, msg) \
+    if (!(x)) { \
+        printf("FAIL:%s:%d:%s\n", __FILE__, __LINE__, (msg)); \
+        printf("TEST_FINISH: FAIL\n"); \
+        test_result = false; \
+        vTaskDelete(NULL); \
+        taskYIELD(); \
+    } 
+
+#define TEST_FAIL(msg) TEST_ASSERT(false, msg)
+
+#define TEST_FINISH() \
+    if (test_result) { \
+        printf("TEST_FINISH: SUCCESS\n"); \
+    } else { \
+        printf("TEST_FINISH: FAIL\n"); \
+    }
+
+#define TEST_INIT(timeout) \
+    if (true) { \
+        test_result = true; \
+        uart_set_baud(0, 115200); \
+        printf("TEST_INIT: TIMEOUT=%ds\n", timeout); \
+    }
+
+
+#endif  // __TEST_H__

tests/Makefile

@@ -0,0 +1,23 @@
+TESTS = $(shell find $(dir $(lastword $(MAKEFILE_LIST))) \
+		-mindepth 2 -maxdepth 2 -name Makefile | sed s/Makefile//g)
+
+# Generate some dummy .dummybuild/.dummyrebuild target files
+TESTS_BUILD = $(patsubst %,%.dummybuild,$(TESTS))
+TESTS_TEST = $(patsubst %,%.dummytest,$(TESTS))
+
+all: $(TESTS_BUILD)
+
+$(TESTS_TEST): $(TESTS_BUILD)
+
+test: $(TESTS_TEST)
+
+%.dummybuild:
+	$(MAKE) -C $(dir $@)
+
+%.dummytest:
+	$(MAKE) -C $(dir $@) flash
+	$(MAKE) -C $(dir $@) run_test
+
+.PHONY: all test
+.NOTPARALLEL:
+.ONESHELL:

tests/example/Makefile

@@ -0,0 +1,10 @@
+PROGRAM=example_test
+
+EXTRA_COMPONENTS = extras/test
+
+include ../../common.mk
+
+run_test:
+	python ../../extras/test/run_test.py /dev/ttyUSB0
+
+.PHONY: run_test

tests/example/example_test.c

@@ -0,0 +1,25 @@
+#include <stdlib.h>
+#include <string.h>
+#include "FreeRTOS.h"
+#include "task.h"
+
+#include "test/test.h"
+
+void test_task(void *pvParameters)
+{
+    // Test will continue even if check is failed
+    TEST_CHECK(2 + 2 == 4, "math doesn't work");
+
+    // Test will end imidiatelly if assertion is failed
+    TEST_ASSERT(strlen("test") == 4, "strlen doesn't work");
+
+    // Finish the test and report the result
+    TEST_FINISH();
+}
+
+void user_init(void)
+{
+    TEST_INIT(5);   // 5 seconds timeout
+
+    xTaskCreate(test_task, (signed char *)"test_task", 256, NULL, 2, NULL);
+}

tests/spiffs/Makefile

@@ -1,7 +1,7 @@
 PROGRAM=posix_fs_example
 PROGRAM_EXTRA_SRC_FILES=./fs-test/fs_test.c
 
-EXTRA_COMPONENTS = extras/spiffs
+EXTRA_COMPONENTS = extras/spiffs extras/test
 FLASH_SIZE = 32
 
 # spiffs configuration
@@ -9,3 +9,8 @@ SPIFFS_BASE_ADDR = 0x200000
 SPIFFS_SIZE = 0x100000
 
 include ../../common.mk
+
+run_test:
+	python ../../extras/test/run_test.py /dev/ttyUSB0
+
+.PHONY: run_test

tests/spiffs/posix_fs_example.c

@@ -11,6 +11,8 @@
 
 #include "fs-test/fs_test.h"
 
+#include "test/test.h"
+
 static fs_time_t get_current_time()
 {
      return timer_get_count(FRC2) / 5000;  // to get roughly 1ms resolution
@@ -28,28 +30,21 @@ void test_task(void *pvParameters)
     }
     esp_spiffs_mount();
 
-    while (1) {
-        vTaskDelay(5000 / portTICK_RATE_MS);
-        if (fs_load_test_run(100)) {
-            printf("PASS\n");
-        } else {
-            printf("FAIL\n");
-        }
+    TEST_CHECK(fs_load_test_run(100), "Load test failed");
 
-        vTaskDelay(5000 / portTICK_RATE_MS);
-        float write_rate, read_rate;
-        if (fs_speed_test_run(get_current_time, &write_rate, &read_rate)) {
-            printf("Read speed: %.0f bytes/s\n", read_rate * 1000); 
-            printf("Write speed: %.0f bytes/s\n", write_rate * 1000); 
-        } else {
-            printf("FAIL\n");
-        }
+    float write_rate, read_rate;
+    if (fs_speed_test_run(get_current_time, &write_rate, &read_rate)) {
+        printf("Read speed: %.0f bytes/s\n", read_rate * 1000); 
+        printf("Write speed: %.0f bytes/s\n", write_rate * 1000); 
+    } else {
+        TEST_FAIL("Speed test failed");
     }
+    TEST_FINISH();
 }
 
 void user_init(void)
 {
-    uart_set_baud(0, 115200);
+    TEST_INIT(60);
 
     xTaskCreate(test_task, (signed char *)"test_task", 1024, NULL, 2, NULL);
 }