Travis running tests on remote servers.

Extend test_runner.py to flash using esptool.py

common.mk

@@ -136,7 +136,7 @@ $$($(1)_OBJ_DIR)%.o: $$($(1)_REAL_ROOT)%.S $$($(1)_MAKEFILE) $(wildcard $(ROOT)*
 
 $(1)_AR_IN_FILES = $$($(1)_OBJ_FILES)
 
-# the component is shown to depend on both obj and source files so we get 
+# the component is shown to depend on both obj and source files so we get
 # a meaningful error message for missing explicitly named source files
 ifeq ($(INCLUDE_SRC_INTO_AR),1)
    $(1)_SRC_IN_AR_FILES = $$($(1)_SRC_FILES)
@@ -215,9 +215,14 @@ $(FW_FILE): $(PROGRAM_OUT) $(FIRMWARE_DIR)
 	$(vecho) "FW $@"
 	$(Q) $(ESPTOOL) elf2image --version=2 $(ESPTOOL_ARGS) $< -o $(FW_FILE)
 
+ESPTOOL_FLASH_CMD ?= -p $(ESPPORT) --baud $(ESPBAUD) write_flash $(ESPTOOL_ARGS) \
+	0x0 $(RBOOT_BIN) 0x1000 $(RBOOT_CONF) 0x2000 $(FW_FILE) $(SPIFFS_ESPTOOL_ARGS)
+
 flash: all
-	$(ESPTOOL) -p $(ESPPORT) --baud $(ESPBAUD) write_flash $(ESPTOOL_ARGS) \
+	$(Q) $(ESPTOOL) $(ESPTOOL_FLASH_CMD)
-		0x0 $(RBOOT_BIN) 0x1000 $(RBOOT_CONF) 0x2000 $(FW_FILE) $(SPIFFS_ESPTOOL_ARGS)
+
+print_flash_cmd:
+	$(Q) echo "$(ESPTOOL_FLASH_CMD)"
 
 erase_flash:
 	$(ESPTOOL) -p $(ESPPORT) --baud $(ESPBAUD) erase_flash
@@ -262,6 +267,9 @@ help:
 	@echo "test"
 	@echo "'flash', then start a GNU Screen session on the same serial port to see serial output."
 	@echo ""
+	@echo "print_flash_cmd"
+	@echo "Just print command line arguments for flashing with esptool.py"
+	@echo ""
 	@echo "size"
 	@echo "Build, then print a summary of built firmware size."
 	@echo ""

tests/README.md

@@ -39,6 +39,12 @@ If not specified device `/dev/ttyUSB1` is used.
 
 `--no-flash` or `-n` - Do not flash the test firmware before running tests.
 
+`--flash` or `-f` - Flash device directly with esptool instead of using
+`make flash` command. Can be used to flash binaries without esp-open-rtos
+environment.
+
+`--flash-cmd` or `-c` - Flash command for esptool. Used together with `--flash`.
+
 `--list` or `-l` - Display list of the available test cases on the device.
 
 ### Example

tests/test_runner.py

@@ -44,9 +44,9 @@ def main():
     verbose = args.verbose
 
     if not args.no_flash:
-        flash_image(args.aport)
+        flash(args.aport, args)
         if args.type != 'solo':
-            flash_image(args.bport)
+            flash(args.bport, args)
 
     env = TestEnvironment(args.aport, TestEnvironment.A)
     env_b = None
@@ -279,11 +279,34 @@ def get_testdir():
     Return the 'tests' directory in the source tree
     (assuming the test_runner.py script is in that directory.
     """
-    res = os.path.dirname(__name__)
+    res = os.path.dirname(__file__)
     return "." if res == "" else res
 
 
-def flash_image(serial_port):
+def flash(serial_port, args):
+    if args.flash:
+        esptool_flash(serial_port, args.flash_cmd)
+    else:
+        make_flash(serial_port)
+
+
+def esptool_flash(serial_port, params):
+    env = dict(os.environ)
+    verbose_print("Flashing test image to %s..." % serial_port)
+    try:
+        stdout = sys.stdout if verbose else None
+        cmd = ["esptool.py", "-p", serial_port]
+        cmd.extend(params.split(' '))
+        cmd = [x for x in cmd if x]  # remove empty elements
+        subprocess.check_call(cmd, cwd=get_testdir(), stdout=stdout,
+                              stderr=subprocess.STDOUT, env=env)
+    except subprocess.CalledProcessError as e:
+        raise TestRunnerError("'esptool.py flash serial=%s' failed with exit code %d" %
+                              (serial_port, e.returncode))
+    verbose_print("Flashing successful.")
+
+
+def make_flash(serial_port):
     # Bit hacky: rather than calling esptool directly,
     # just use the Makefile flash target with the correct ESPPORT argument
     env = dict(os.environ)
@@ -329,6 +352,17 @@ def parse_args():
         action='store_true',
         default=False)
 
+    parser.add_argument(
+        '--flash', '-f',
+        help='Flash device directly with esptool',
+        action='store_true',
+        default=False)
+
+    parser.add_argument(
+        '--flash-cmd', '-c',
+        help='Flash command for esptool',
+        default='write_flash 0x2000 ./firmware/tests.bin')
+
     parser.add_argument(
         '--verbose', '-v',
         help='Verbose test runner debugging output',

utils/travis_tests/run_tests.sh

@@ -1,13 +1,58 @@
 #!/bin/bash
+#
+# This script builds tests, deploys them on one of the available test
+# servers and runs them.
+#
 
-TEST_SERVERS[0]="IP=195.138.84.66;User=pi;Type=dual"
+# Test servers configuration
-# TEST_SERVERS[1]="IP=195.138.84.55;Type=solo"
+TEST_SERVERS[0]="IP=195.138.84.66;User=pi;Type=solo"
+TEST_SERVERS[1]="IP=195.138.84.66;User=pi;Type=dual"
 
+FLASH_CMD=
+
+# Function doesn't accept any arguments. It build the tests,
+# packages the binaries into the archive and populates FLASH_CMD variable.
 function build {
     echo "Building tests"
     make -C ./tests clean
     make -C ./tests -j8
-    tar -czf /tmp/tests.tar.gz ./tests ./common.mk ./parameters.mk
+    FLASH_CMD=$(make -C ./tests print_flash_cmd)
+
+    # Now we need to pack all files that are included in the flash cmd
+    # so they can be transfered to the remote server and run there
+    # Also we need to prepare flash command:
+    #  - remove firmware files path
+    #  - remove serial port parameter
+    mkdir -p /tmp/firmware
+    rm -rf /tmp/firmware/*
+    params=($FLASH_CMD)
+    pushd ./tests
+    for param in "${params[@]}"
+    do
+        if [ -f ${param} ]
+        then
+            file_name=${param##*/}
+            cp ${param} /tmp/firmware/
+            FLASH_CMD=${FLASH_CMD/${param}/${file_name}}
+        fi
+
+        # Removing port parameter from the cmd string
+        if [[ "$param" == "-p" || "$param" == "--port" ]]
+        then
+            FLASH_CMD=${FLASH_CMD/${param}/}
+            next_port=true
+        else
+            # Removing port value from the cmd string
+            if [ "$next_port" ]
+            then
+                FLASH_CMD=${FLASH_CMD/${param} /}
+                unset next_port
+            fi
+        fi
+    done
+    cp test_runner.py /tmp/firmware/
+    tar -czf /tmp/tests.tar.gz -C /tmp/firmware .
+    popd
 }
 
 # $1 - Server IP
@@ -24,19 +69,36 @@ function deploy {
 # $2 - Login user name
 # $3 - Type "solo" or "dual"
 function run_tests {
-    echo "Running tests, server IP=${1}, type=${2}"
+    echo "Running tests, server IP=${1}, type=${3}"
+    echo "Flash cmd: ${FLASH_CMD}"
+    # Run test runner on the remote server
+    # ssh ${2}@${1} "source /home/pi/.profile; cd /tmp/eor_test/; ./test_runner.py --type ${3} -f -c \"${FLASH_CMD}\""
+    ssh ${2}@${1} "source /home/pi/.profile; /tmp/eor_test/test_runner.py --type ${3} -f -c \"${FLASH_CMD}\""
 }
 
+# First step is to build a firmware
 build
 
+failed=0
+
 for server in "${TEST_SERVERS[@]}"
 do
     params=(${server//;/ })
     ip=${params[0]#IP=}
     user=${params[1]#User=}
     type=${params[2]#Type=}
-    
+
     deploy ${ip} ${user}
-    run_tests ${ip} ${user} ${type}
+    if [ "$?" -eq "0" ]
+    then
+        run_tests ${ip} ${user} ${type}
+        if [ "$?" -ne "0" ]
+        then
+            failed=$((failed+1))
+        fi
+    else
+        echo "Server ${ip} is not available"
+    fi
 done
 
+exit $failed