diff --git a/.travis.yml b/.travis.yml index c00c5d2..9841d86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ cache: directories: - ${CROSS_ROOT} addons: + ssh_known_hosts: 195.138.84.66 apt: packages: - make @@ -35,6 +36,11 @@ addons: - vim-common before_install: + - openssl aes-256-cbc -K $encrypted_709d8233e262_key -iv $encrypted_709d8233e262_iv + -in utils/travis_tests/sheinz_rsa.enc -out /tmp/sheinz_rsa -d + - eval "$(ssh-agent -s)" + - chmod 600 /tmp/sheinz_rsa + - ssh-add /tmp/sheinz_rsa - travis_wait 30 utils/travis_build/install_toolchain.sh script: @@ -45,3 +51,5 @@ script: - ( ${MAKE_CMD} ) || ( ${MAKE_CMD} V=1 ) # build bootloader - make -C bootloader/ + # run tests + - ./utils/travis_tests/run_tests.sh diff --git a/common.mk b/common.mk index b35dbd9..8950893 100644 --- a/common.mk +++ b/common.mk @@ -146,7 +146,7 @@ ifndef $(1)_WHOLE_ARCHIVE $(1)_AR_IN_FILES += $$($(1)_SRC_FILES) endif -$$($(1)_AR): $$($(1)_AR_IN_FILES) +$$($(1)_AR): $$($(1)_OBJ_FILES) $$($(1)_SRC_IN_AR_FILES) $(vecho) "AR $$@" $(Q) mkdir -p $$(dir $$@) $(Q) $(AR) cru $$@ $$^ @@ -223,9 +223,14 @@ $(FW_FILE): $(PROGRAM_OUT) $(FIRMWARE_DIR) $(vecho) "FW $@" $(Q) $(ESPTOOL) elf2image --version=2 $(ESPTOOL_ARGS) $< -o $(FW_FILE) +ESPTOOL_FLASH_CMD ?= -p $(ESPPORT) --baud $(ESPBAUD) write_flash $(ESPTOOL_ARGS) \ + 0x0 $(RBOOT_BIN) 0x1000 $(RBOOT_CONF) 0x2000 $(FW_FILE) $(SPIFFS_ESPTOOL_ARGS) + flash: all - $(ESPTOOL) -p $(ESPPORT) --baud $(ESPBAUD) write_flash $(ESPTOOL_ARGS) \ - 0x0 $(RBOOT_BIN) 0x1000 $(RBOOT_CONF) 0x2000 $(FW_FILE) $(SPIFFS_ESPTOOL_ARGS) + $(Q) $(ESPTOOL) $(ESPTOOL_FLASH_CMD) + +print_flash_cmd: + $(Q) echo "$(ESPTOOL_FLASH_CMD)" erase_flash: $(ESPTOOL) -p $(ESPPORT) --baud $(ESPBAUD) erase_flash @@ -270,6 +275,9 @@ help: @echo "test" @echo "'flash', then start a GNU Screen session on the same serial port to see serial output." @echo "" + @echo "print_flash_cmd" + @echo "Just print command line arguments for flashing with esptool.py" + @echo "" @echo "size" @echo "Build, then print a summary of built firmware size." @echo "" diff --git a/tests/README.md b/tests/README.md index 90b08a8..a147049 100644 --- a/tests/README.md +++ b/tests/README.md @@ -39,6 +39,12 @@ If not specified device `/dev/ttyUSB1` is used. `--no-flash` or `-n` - Do not flash the test firmware before running tests. +`--flash` or `-f` - Flash device directly with esptool instead of using +`make flash` command. Can be used to flash binaries without esp-open-rtos +environment. + +`--flash-cmd` or `-c` - Flash command for esptool. Used together with `--flash`. + `--list` or `-l` - Display list of the available test cases on the device. ### Example diff --git a/tests/test_runner.py b/tests/test_runner.py index 347fd9b..7f7a440 100755 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -44,9 +44,9 @@ def main(): verbose = args.verbose if not args.no_flash: - flash_image(args.aport) + flash(args.aport, args) if args.type != 'solo': - flash_image(args.bport) + flash(args.bport, args) env = TestEnvironment(args.aport, TestEnvironment.A) env_b = None @@ -279,11 +279,34 @@ def get_testdir(): Return the 'tests' directory in the source tree (assuming the test_runner.py script is in that directory. """ - res = os.path.dirname(__name__) + res = os.path.dirname(__file__) return "." if res == "" else res -def flash_image(serial_port): +def flash(serial_port, args): + if args.flash: + esptool_flash(serial_port, args.flash_cmd) + else: + make_flash(serial_port) + + +def esptool_flash(serial_port, params): + env = dict(os.environ) + verbose_print("Flashing test image to %s..." % serial_port) + try: + stdout = sys.stdout if verbose else None + cmd = ["esptool.py", "-p", serial_port] + cmd.extend(params.split(' ')) + cmd = [x for x in cmd if x] # remove empty elements + subprocess.check_call(cmd, cwd=get_testdir(), stdout=stdout, + stderr=subprocess.STDOUT, env=env) + except subprocess.CalledProcessError as e: + raise TestRunnerError("'esptool.py flash serial=%s' failed with exit code %d" % + (serial_port, e.returncode)) + verbose_print("Flashing successful.") + + +def make_flash(serial_port): # Bit hacky: rather than calling esptool directly, # just use the Makefile flash target with the correct ESPPORT argument env = dict(os.environ) @@ -329,6 +352,17 @@ def parse_args(): action='store_true', default=False) + parser.add_argument( + '--flash', '-f', + help='Flash device directly with esptool', + action='store_true', + default=False) + + parser.add_argument( + '--flash-cmd', '-c', + help='Flash command for esptool', + default='write_flash 0x2000 ./firmware/tests.bin') + parser.add_argument( '--verbose', '-v', help='Verbose test runner debugging output', diff --git a/utils/travis_tests/README.md b/utils/travis_tests/README.md new file mode 100644 index 0000000..5cd38dc --- /dev/null +++ b/utils/travis_tests/README.md @@ -0,0 +1,101 @@ +Travis CI Tests +==================== + +This directory contains a script `run_tests.sh` that is executed by Travis CI. +The script builds a test firmware, deploys it on one of the test servers and +runs it. + +The script will not return an error if deployment to one of the test servers has +failed. It is done this way not to fail a build if a test server is down. +The script will return an error if deployment was successful but tests failed. + +Test servers +------------ + +Test server is a linux host that is accessible from the Internet by a static IP. +It should have at least one ESP8266 module connected to a USB port. The module +should be capable restarting and switching to boot mode via a serial port. +All popular **NodeMCU** and **Wemos** modules will work. + +To run tests on a server it should provide SSH access. SSH daemon should be +configured to authenticate using keys. + +Test server running on Raspberry PI: + +![Raspberry PI Test server][example-test-server] + +### Test server requirements + +* Linux host +* Public static IP +* One or two ESP8266 modules connected to USB ports +* SSH access from the Internet (with public key from Travis CI) +* Python3 +* [esptool.py] installed `pip install esptool` +* pySerial python module `pip3 install pyserial` + +### Create SSH keys for Travis + +[Here][travis-ssh-deploy] is a good article about Travis deployment using SSH. + +The problem with SSH access from Travis to a server is that it should have +a private key. But this key should not be publicly available. + +Hopefully Travis allows to encrypt certain files and only decrypt them at build +stage. So the sensitive file is stored in the repository encrypted. + +Generate a new key pair: +```bash +ssh-keygen -t rsa -b 4096 -C '@travis-ci.org' -f ./_rsa +``` + +To encrypt a private key you need a command line Travis client. + +To install it run: +```bash +gem install travis +``` +Or refer [the official installation instructions][travis-install]. + +The following command will encrypt a file and modify .travis.yml: +```bash +travis encrypt-file _rsa --add +``` + +Deploy public key to a test server: +```bash +ssh-copy-id -i _rsa.pub @ +``` + +Add the following lines in the .travis.yml: +```yml +addons: + ssh_known_hosts: +``` +```yml +before_install: +- openssl aes-256-cbc aes-256-cbc -K $encrypted_<...>_key -iv $encrypted_<...>_iv -in _rsa.enc -out /tmp/_rsa -d +- eval "$(ssh-agent -s)" +- chmod 600 /tmp/_rsa +- ssh-add /tmp/_rsa +``` + +Remove keys and stage files for commit: +```bash +rm -f _rsa _rsa.pub +git add _rsa.enc .travis.yml +``` + +### Add test server + +The final step is to add a server to the test runner script. +Add a new item into an array in `run_tests.sh`: +```bash +TEST_SERVERS[2]="IP=;User=;Type=" +``` + + +[esptool.py]: https://github.com/espressif/esptool +[travis-ssh-deploy]: https://oncletom.io/2016/travis-ssh-deploy +[travis-install]: https://github.com/travis-ci/travis.rb#installation +[example-test-server]: ./test_server_example.png diff --git a/utils/travis_tests/run_tests.sh b/utils/travis_tests/run_tests.sh new file mode 100755 index 0000000..96eff7f --- /dev/null +++ b/utils/travis_tests/run_tests.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# +# This script builds tests, deploys them on one of the available test +# servers and runs them. If deployment fails it will not return an error code. +# If tests fail the script will return an error code. +# It is done this way not to fail Travis build if one of the test servers is +# down. + +# Test servers configuration +TEST_SERVERS[0]="IP=195.138.84.66;User=pi;Type=solo" +TEST_SERVERS[1]="IP=195.138.84.66;User=pi;Type=dual" + +# It will be populated in 'build' function +FLASH_CMD= + +# Function doesn't accept any arguments. It builds the tests, +# packages the binaries into the archive and populates FLASH_CMD variable. +function build { + echo "Building tests" + make -C ./tests clean + make -C ./tests -j8 + FLASH_CMD=$(make -s -C ./tests print_flash_cmd) + + # Now we need to pack all files that are included in the flash cmd + # so they can be transferred to the remote server and run there + # Also we need to prepare flash command: + # - remove firmware files path + # - remove serial port parameter + mkdir -p /tmp/firmware + rm -rf /tmp/firmware/* + params=($FLASH_CMD) + pushd ./tests + for param in "${params[@]}" + do + if [ -f ${param} ] + then + file_name=${param##*/} + cp ${param} /tmp/firmware/ + FLASH_CMD=${FLASH_CMD/${param}/${file_name}} + fi + + # Removing port parameter from the cmd string + if [[ "$param" == "-p" || "$param" == "--port" ]] + then + FLASH_CMD=${FLASH_CMD/${param}/} + next_port=true + else + # Removing port value from the cmd string + if [ "$next_port" ] + then + FLASH_CMD=${FLASH_CMD/${param} /} + unset next_port + fi + fi + done + cp test_runner.py /tmp/firmware/ + tar -czf /tmp/tests.tar.gz -C /tmp/firmware . + popd +} + +# $1 - Server IP +# $2 - Login user name +function deploy { + echo "Deploying tests, server IP=${1}" + scp /tmp/tests.tar.gz ${2}@${1}:/tmp/tests.tar.gz + ssh ${2}@${1} mkdir -p /tmp/eor_test + ssh ${2}@${1} rm -rf /tmp/eor_test/* + ssh ${2}@${1} tar -xzf /tmp/tests.tar.gz -C /tmp/eor_test +} + +# $1 - Server IP +# $2 - Login user name +# $3 - Type "solo" or "dual" +function run_tests { + echo "Running tests, server IP=${1}, type=${3}" + echo "Flash cmd: ${FLASH_CMD}" + # Run test runner on the remote server + ssh ${2}@${1} "source ~/.profile; /tmp/eor_test/test_runner.py --type ${3} -f -c \"${FLASH_CMD}\"" +} + +# First step is to build a firmware +build + +failed=0 + +for server in "${TEST_SERVERS[@]}" +do + params=(${server//;/ }) + ip=${params[0]#IP=} + user=${params[1]#User=} + type=${params[2]#Type=} + + deploy ${ip} ${user} + if [ "$?" -eq "0" ] + then + run_tests ${ip} ${user} ${type} + if [ "$?" -ne "0" ] + then + failed=$((failed+1)) + fi + else + echo "Server ${ip} is not available" + fi +done + +exit $failed diff --git a/utils/travis_tests/sheinz_rsa.enc b/utils/travis_tests/sheinz_rsa.enc new file mode 100644 index 0000000..65cc620 Binary files /dev/null and b/utils/travis_tests/sheinz_rsa.enc differ diff --git a/utils/travis_tests/test_server_example.png b/utils/travis_tests/test_server_example.png new file mode 100644 index 0000000..4f93f19 Binary files /dev/null and b/utils/travis_tests/test_server_example.png differ