Travis running tests on test servers (#272)

- README.md for adding test servers
 - Bash script for running tests on remote servers
 - Extend test_runner.py to flash using esptool.py
 - Python3.4 support. README.md update

.travis.yml

@@ -12,6 +12,7 @@ cache:
   directories:
     - ${CROSS_ROOT}
 addons:
+  ssh_known_hosts: 195.138.84.66
   apt:
     packages:
     - make
@@ -35,6 +36,11 @@ addons:
     - vim-common
 
 before_install:
+  - openssl aes-256-cbc -K $encrypted_709d8233e262_key -iv $encrypted_709d8233e262_iv
+    -in utils/travis_tests/sheinz_rsa.enc -out /tmp/sheinz_rsa -d
+  - eval "$(ssh-agent -s)"
+  - chmod 600 /tmp/sheinz_rsa
+  - ssh-add /tmp/sheinz_rsa
   - travis_wait 30 utils/travis_build/install_toolchain.sh
 
 script:
@@ -45,3 +51,5 @@ script:
   - ( ${MAKE_CMD} ) || ( ${MAKE_CMD} V=1 )
   # build bootloader
   - make -C bootloader/
+  # run tests
+  - ./utils/travis_tests/run_tests.sh

common.mk

@@ -146,7 +146,7 @@ ifndef $(1)_WHOLE_ARCHIVE
    $(1)_AR_IN_FILES += $$($(1)_SRC_FILES)
 endif
 
-$$($(1)_AR): $$($(1)_AR_IN_FILES)
+$$($(1)_AR): $$($(1)_OBJ_FILES) $$($(1)_SRC_IN_AR_FILES)
 	$(vecho) "AR $$@"
 	$(Q) mkdir -p $$(dir $$@)
 	$(Q) $(AR) cru $$@ $$^
@@ -223,10 +223,15 @@ $(FW_FILE): $(PROGRAM_OUT) $(FIRMWARE_DIR)
 	$(vecho) "FW $@"
 	$(Q) $(ESPTOOL) elf2image --version=2 $(ESPTOOL_ARGS) $< -o $(FW_FILE)
 
-flash: all
-	$(ESPTOOL) -p $(ESPPORT) --baud $(ESPBAUD) write_flash $(ESPTOOL_ARGS) \
+ESPTOOL_FLASH_CMD ?= -p $(ESPPORT) --baud $(ESPBAUD) write_flash $(ESPTOOL_ARGS) \
 	0x0 $(RBOOT_BIN) 0x1000 $(RBOOT_CONF) 0x2000 $(FW_FILE) $(SPIFFS_ESPTOOL_ARGS)
 
+flash: all
+	$(Q) $(ESPTOOL) $(ESPTOOL_FLASH_CMD)
+
+print_flash_cmd:
+	$(Q) echo "$(ESPTOOL_FLASH_CMD)"
+
 erase_flash:
 	$(ESPTOOL) -p $(ESPPORT) --baud $(ESPBAUD) erase_flash
 
@@ -270,6 +275,9 @@ help:
 	@echo "test"
 	@echo "'flash', then start a GNU Screen session on the same serial port to see serial output."
 	@echo ""
+	@echo "print_flash_cmd"
+	@echo "Just print command line arguments for flashing with esptool.py"
+	@echo ""
 	@echo "size"
 	@echo "Build, then print a summary of built firmware size."
 	@echo ""

tests/README.md

@@ -39,6 +39,12 @@ If not specified device `/dev/ttyUSB1` is used.
 
 `--no-flash` or `-n` - Do not flash the test firmware before running tests.
 
+`--flash` or `-f` - Flash device directly with esptool instead of using
+`make flash` command. Can be used to flash binaries without esp-open-rtos
+environment.
+
+`--flash-cmd` or `-c` - Flash command for esptool. Used together with `--flash`.
+
 `--list` or `-l` - Display list of the available test cases on the device.
 
 ### Example

tests/test_runner.py

@@ -44,9 +44,9 @@ def main():
     verbose = args.verbose
 
     if not args.no_flash:
-        flash_image(args.aport)
+        flash(args.aport, args)
         if args.type != 'solo':
-            flash_image(args.bport)
+            flash(args.bport, args)
 
     env = TestEnvironment(args.aport, TestEnvironment.A)
     env_b = None
@@ -279,11 +279,34 @@ def get_testdir():
     Return the 'tests' directory in the source tree
     (assuming the test_runner.py script is in that directory.
     """
-    res = os.path.dirname(__name__)
+    res = os.path.dirname(__file__)
     return "." if res == "" else res
 
 
-def flash_image(serial_port):
+def flash(serial_port, args):
+    if args.flash:
+        esptool_flash(serial_port, args.flash_cmd)
+    else:
+        make_flash(serial_port)
+
+
+def esptool_flash(serial_port, params):
+    env = dict(os.environ)
+    verbose_print("Flashing test image to %s..." % serial_port)
+    try:
+        stdout = sys.stdout if verbose else None
+        cmd = ["esptool.py", "-p", serial_port]
+        cmd.extend(params.split(' '))
+        cmd = [x for x in cmd if x]  # remove empty elements
+        subprocess.check_call(cmd, cwd=get_testdir(), stdout=stdout,
+                              stderr=subprocess.STDOUT, env=env)
+    except subprocess.CalledProcessError as e:
+        raise TestRunnerError("'esptool.py flash serial=%s' failed with exit code %d" %
+                              (serial_port, e.returncode))
+    verbose_print("Flashing successful.")
+
+
+def make_flash(serial_port):
     # Bit hacky: rather than calling esptool directly,
     # just use the Makefile flash target with the correct ESPPORT argument
     env = dict(os.environ)
@@ -329,6 +352,17 @@ def parse_args():
         action='store_true',
         default=False)
 
+    parser.add_argument(
+        '--flash', '-f',
+        help='Flash device directly with esptool',
+        action='store_true',
+        default=False)
+
+    parser.add_argument(
+        '--flash-cmd', '-c',
+        help='Flash command for esptool',
+        default='write_flash 0x2000 ./firmware/tests.bin')
+
     parser.add_argument(
         '--verbose', '-v',
         help='Verbose test runner debugging output',

utils/travis_tests/README.md

@@ -0,0 +1,101 @@
+Travis CI Tests
+====================
+
+This directory contains a script `run_tests.sh` that is executed by Travis CI.
+The script builds a test firmware, deploys it on one of the test servers and
+runs it.
+
+The script will not return an error if deployment to one of the test servers has
+failed. It is done this way not to fail a build if a test server is down.
+The script will return an error if deployment was successful but tests failed.
+
+Test servers
+------------
+
+Test server is a linux host that is accessible from the Internet by a static IP.
+It should have at least one ESP8266 module connected to a USB port. The module
+should be capable restarting and switching to boot mode via a serial port.
+All popular **NodeMCU** and **Wemos** modules will work.
+
+To run tests on a server it should provide SSH access. SSH daemon should be
+configured to authenticate using keys.
+
+Test server running on Raspberry PI:
+
+![Raspberry PI Test server][example-test-server]
+
+### Test server requirements
+
+* Linux host
+* Public static IP
+* One or two ESP8266 modules connected to USB ports
+* SSH access from the Internet (with public key from Travis CI)
+* Python3
+* [esptool.py] installed `pip install esptool`
+* pySerial python module `pip3 install pyserial`
+
+### Create SSH keys for Travis
+
+[Here][travis-ssh-deploy] is a good article about Travis deployment using SSH.
+
+The problem with SSH access from Travis to a server is that it should have
+a private key. But this key should not be publicly available.
+
+Hopefully Travis allows to encrypt certain files and only decrypt them at build
+stage. So the sensitive file is stored in the repository encrypted.
+
+Generate a new key pair:
+```bash
+ssh-keygen -t rsa -b 4096 -C '<repo>@travis-ci.org' -f ./<server_name>_rsa
+```
+
+To encrypt a private key you need a command line Travis client.
+
+To install it run:
+```bash
+gem install travis
+```
+Or refer [the official installation instructions][travis-install].
+
+The following command will encrypt a file and modify .travis.yml:
+```bash
+travis encrypt-file <server_name>_rsa --add
+```
+
+Deploy public key to a test server:
+```bash
+ssh-copy-id -i <server_name>_rsa.pub <ssh-user>@<deploy-host>
+```
+
+Add the following lines in the .travis.yml:
+```yml
+addons:
+  ssh_known_hosts: <server_ip>
+```
+```yml
+before_install:
+- openssl aes-256-cbc aes-256-cbc -K $encrypted_<...>_key -iv $encrypted_<...>_iv -in <server_name>_rsa.enc -out /tmp/<server_name>_rsa -d
+- eval "$(ssh-agent -s)"
+- chmod 600 /tmp/<server_name>_rsa
+- ssh-add /tmp/<server_name>_rsa
+```
+
+Remove keys and stage files for commit:
+```bash
+rm -f <server_name>_rsa <server_name>_rsa.pub
+git add <server_name>_rsa.enc .travis.yml
+```
+
+### Add test server
+
+The final step is to add a server to the test runner script.
+Add a new item into an array in `run_tests.sh`:
+```bash
+TEST_SERVERS[2]="IP=<server_ip>;User=<ssh_user_name>;Type=<solo|dual>"
+```
+
+
+[esptool.py]: https://github.com/espressif/esptool
+[travis-ssh-deploy]: https://oncletom.io/2016/travis-ssh-deploy
+[travis-install]: https://github.com/travis-ci/travis.rb#installation
+[example-test-server]: ./test_server_example.png

utils/travis_tests/run_tests.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+#
+# This script builds tests, deploys them on one of the available test
+# servers and runs them. If deployment fails it will not return an error code.
+# If tests fail the script will return an error code.
+# It is done this way not to fail Travis build if one of the test servers is
+# down.
+
+# Test servers configuration
+TEST_SERVERS[0]="IP=195.138.84.66;User=pi;Type=solo"
+TEST_SERVERS[1]="IP=195.138.84.66;User=pi;Type=dual"
+
+# It will be populated in 'build' function
+FLASH_CMD=
+
+# Function doesn't accept any arguments. It builds the tests,
+# packages the binaries into the archive and populates FLASH_CMD variable.
+function build {
+    echo "Building tests"
+    make -C ./tests clean
+    make -C ./tests -j8
+    FLASH_CMD=$(make -s -C ./tests print_flash_cmd)
+
+    # Now we need to pack all files that are included in the flash cmd
+    # so they can be transferred to the remote server and run there
+    # Also we need to prepare flash command:
+    #  - remove firmware files path
+    #  - remove serial port parameter
+    mkdir -p /tmp/firmware
+    rm -rf /tmp/firmware/*
+    params=($FLASH_CMD)
+    pushd ./tests
+    for param in "${params[@]}"
+    do
+        if [ -f ${param} ]
+        then
+            file_name=${param##*/}
+            cp ${param} /tmp/firmware/
+            FLASH_CMD=${FLASH_CMD/${param}/${file_name}}
+        fi
+
+        # Removing port parameter from the cmd string
+        if [[ "$param" == "-p" || "$param" == "--port" ]]
+        then
+            FLASH_CMD=${FLASH_CMD/${param}/}
+            next_port=true
+        else
+            # Removing port value from the cmd string
+            if [ "$next_port" ]
+            then
+                FLASH_CMD=${FLASH_CMD/${param} /}
+                unset next_port
+            fi
+        fi
+    done
+    cp test_runner.py /tmp/firmware/
+    tar -czf /tmp/tests.tar.gz -C /tmp/firmware .
+    popd
+}
+
+# $1 - Server IP
+# $2 - Login user name
+function deploy {
+    echo "Deploying tests, server IP=${1}"
+    scp /tmp/tests.tar.gz ${2}@${1}:/tmp/tests.tar.gz
+    ssh ${2}@${1} mkdir -p /tmp/eor_test
+    ssh ${2}@${1} rm -rf /tmp/eor_test/*
+    ssh ${2}@${1} tar -xzf /tmp/tests.tar.gz -C /tmp/eor_test
+}
+
+# $1 - Server IP
+# $2 - Login user name
+# $3 - Type "solo" or "dual"
+function run_tests {
+    echo "Running tests, server IP=${1}, type=${3}"
+    echo "Flash cmd: ${FLASH_CMD}"
+    # Run test runner on the remote server
+    ssh ${2}@${1} "source ~/.profile; /tmp/eor_test/test_runner.py --type ${3} -f -c \"${FLASH_CMD}\""
+}
+
+# First step is to build a firmware
+build
+
+failed=0
+
+for server in "${TEST_SERVERS[@]}"
+do
+    params=(${server//;/ })
+    ip=${params[0]#IP=}
+    user=${params[1]#User=}
+    type=${params[2]#Type=}
+
+    deploy ${ip} ${user}
+    if [ "$?" -eq "0" ]
+    then
+        run_tests ${ip} ${user} ${type}
+        if [ "$?" -ne "0" ]
+        then
+            failed=$((failed+1))
+        fi
+    else
+        echo "Server ${ip} is not available"
+    fi
+done
+
+exit $failed