Add error handling to OTA process

firmware/otaflash.py

@@ -4,40 +4,68 @@ import time
 import websocket
 import argparse
 import zlib
+from websocket import WebSocketTimeoutException
 
 parser = argparse.ArgumentParser(description='Update fiatlux firmware via websocket.')
 parser.add_argument("binfile")
 
 args = parser.parse_args()
 
+bs = 1024
+
+
+def parse_reply(bytes):
+    cmd = bytes[0:1].decode("utf-8")
+    ret = int.from_bytes(bytes[1:2], "big")
+    val = int.from_bytes(bytes[2:4], "big")
+    return {'cmd': cmd, 'ret': ret, 'val': val}
+
+
 with open(args.binfile, "rb") as f:
     try:
         ws = websocket.WebSocket()
+        print("send {}".format(args.binfile))
         ws.connect("ws://172.16.0.1")
         i = 0
-        rolling = 0
-        total = 0
+        bytes = f.read()
+        rolling = zlib.crc32(bytes)
+        total = len(bytes)
         while True:
-            bytes = f.read(512)
-            rolling = zlib.crc32(bytes, rolling)
-            total += len(bytes)
+            chunk = bytes[bs * i:bs * i + bs]
             msg = b'F\x00\x00\x00'
             msg += i.to_bytes(2, 'big')
-            msg += len(bytes).to_bytes(2, 'big')
-            msg += (zlib.crc32(bytes) & 0xffffffff).to_bytes(4, 'big')
-            msg += bytes
+            msg += len(chunk).to_bytes(2, 'big')
+            msg += (zlib.crc32(chunk) & 0xffffffff).to_bytes(4, 'big')
+            msg += chunk
             ws.send(msg)
-            reply = ws.recv()
+            print("\r{:6.2f}%".format(100 * i * bs / total), end='')
+            reply = parse_reply(ws.recv())
+            if reply['cmd'] == 'F' and reply['ret'] == 0:
+                i += 1
+            elif reply['cmd'] == 'F' and reply['ret'] == 1:
+                print("Error: SEQUENCE_OUT_OF_ORDER")
+                i = reply['val']
+            elif reply['cmd'] == 'F' and reply['ret'] == 2:
+                print("Error: CHECKSUM_MISMATCH")
+                i = reply['val']
+            else:
+                print(reply)
+
             time.sleep(0.05)
-            i += 1
-            if len(bytes) != 512:
+            if len(chunk) != bs:
                 break
+
+        print("\rdone      ")
         msg = b'C\x00\x00\x00'
         msg += total.to_bytes(4, 'big')
-        msg += rolling.to_bytes(4, 'big')
+        msg += (rolling).to_bytes(4, 'big')
+        ws.settimeout(5)
         ws.send(msg)
-        print(ws.recv())
+        reply = parse_reply(ws.recv())
+        print(reply)
         ws.close()
+    except WebSocketTimeoutException:
+        pass
     except ConnectionResetError:
         pass
     except KeyboardInterrupt:

firmware/system.c

@@ -68,7 +68,7 @@ void system_otaflash_init() {
     otaflash_context.seq = 0;
 }
 
-int system_otaflash_chunk(uint8_t *data, uint16_t len, uint16_t seq, uint32_t hash) {
+enum return_code system_otaflash_chunk(uint8_t *data, uint16_t len, uint16_t seq, uint32_t hash, uint16_t *ack) {
     uint32_t local_hash = crc32(data, len);
     if(hash == local_hash && otaflash_context.seq == seq) {
         if(otaflash_context.head % SECTOR_SIZE == 0) {
@@ -83,9 +83,13 @@ int system_otaflash_chunk(uint8_t *data, uint16_t len, uint16_t seq, uint32_t ha
         }
         otaflash_context.head += len;
         otaflash_context.seq++;
-        return 0x88;
+        return OK;
+    } else if(hash != local_hash) {
+        return CHECKSUM_MISMATCH;
     } else {
-        return 0xff;
+        if(ack)
+            *ack = otaflash_context.seq;
+        return SEQUENCE_OUT_OF_ORDER;
     }
 
 }
@@ -96,21 +100,22 @@ void system_otaflash_verify_chunk(void *ctx, void *data, size_t len) {
     *(uint32_t *) ctx = digest;
 }
 
-int system_otaflash_verify_and_switch(uint32_t len, uint32_t hash) {
+enum return_code system_otaflash_verify_and_switch(uint32_t len, uint32_t hash) {
 
     uint32_t digest = 0;
     rboot_digest_image(otaflash_context.base, min(len, MAX_IMAGE_SIZE), system_otaflash_verify_chunk, &digest);
 
     if(hash != digest) {
         printf("OTA failed to verify firmware\r\n");
-        return 0x99;
+        return CHECKSUM_MISMATCH;
     }
 
     vPortEnterCritical();
     if(!rboot_set_current_rom(otaflash_context.slot)) {
-        printf("OTA failed to set new rboot slot\r\n");
+        printf("OTA Update failed to set new rboot slot\r\n");
+        vPortExitCritical();
+        return RBOOT_SWITCH_FAILED;
     }
-    sdk_system_restart();
-    vPortExitCritical(); // | should not be reached
-    return 0x77;         // |
+    vPortExitCritical();
+    return OK;
 }

firmware/system.h

@@ -6,6 +6,7 @@
 #define FIRMWARE_SYSTEM_H
 
 #include <stdint.h>
+#include "types.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -17,9 +18,9 @@ void system_init_config();
 
 void system_otaflash_init();
 
-int system_otaflash_chunk(uint8_t *data, uint16_t len, uint16_t seq, uint32_t hash);
+enum return_code system_otaflash_chunk(uint8_t *data, uint16_t len, uint16_t seq, uint32_t hash, uint16_t *ack);
 
-int system_otaflash_verify_and_switch(uint32_t len, uint32_t hash);
+enum return_code system_otaflash_verify_and_switch(uint32_t len, uint32_t hash);
 
 #ifdef __cplusplus
 }

firmware/types.h

@@ -0,0 +1,20 @@
+//
+// Created by jedi on 09.09.21.
+//
+
+#ifndef FIRMWARE_TYPES_H
+#define FIRMWARE_TYPES_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum return_code {
+    OK = 0, SEQUENCE_OUT_OF_ORDER, CHECKSUM_MISMATCH, RBOOT_SWITCH_FAILED, ERROR = 0xFF
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif //FIRMWARE_TYPES_H

firmware/web.cpp

@@ -215,38 +215,48 @@ struct fw_check {
 void websocket_cb(struct tcp_pcb *pcb, char *data, u16_t data_len,
                   uint8_t /*mode*/) { //mode should be WS_BIN_MODE or WS_TEXT_MODE
 
-    uint8_t response[3];
-    uint16_t val = 0;
-    char cmd = '0';
+    uint8_t response[4];
+    auto &cmd = (char &) response[0];
+    auto &ret = response[1];
+    auto &val = (uint16_t &) response[2];
+    cmd = '0';
+    ret = ERROR;
+    val = 0;
 
-    bool togl = 0;
+    bool togl = false;
 
     switch (data[0]) {
         case 'R': // Restart
             cmd = 'R';
+            ret = OK;
             break;
         case 'X': // Clear Config
             cmd = 'X';
+            ret = OK;
             break;
         case 'D': // Disable LED
             signal_led(false);
-            val = 1;
             cmd = 'G';
+            ret = OK;
+            val = 1;
             break;
         case 'E': // Enable LED
             signal_led(true);
-            val = 0;
             cmd = 'G';
+            ret = OK;
+            val = 0;
             break;
         case 'F':
-            togl = ~togl;
+            togl = !togl;
             signal_led(togl);
             {
                 auto *f = (fw_frame *) data;
                 if(f->seq == 0) {
                     system_otaflash_init();
                 }
-                val = system_otaflash_chunk(f->data, ntohs(f->len), ntohs(f->seq), ntohl(f->hash));
+                uint16_t ack = 0;
+                ret = system_otaflash_chunk(f->data, ntohs(f->len), ntohs(f->seq), ntohl(f->hash), &ack);
+                val = htons(ack);
             }
             cmd = 'F';
             break;
@@ -254,32 +264,31 @@ void websocket_cb(struct tcp_pcb *pcb, char *data, u16_t data_len,
             signal_led(false);
             {
                 auto *f = (fw_check *) data;
-                val = system_otaflash_verify_and_switch(ntohl(f->len), ntohl(f->hash));
+                ret = system_otaflash_verify_and_switch(ntohl(f->len), ntohl(f->hash));
             }
             cmd = 'C';
             break;
         default:
             printf("[websocket_callback]:\n%.*s\n", (int) data_len, (char *) data);
             printf("Unknown command %c\n", data[0]);
-            val = 0;
+            ret = ERROR;
             break;
     }
 
-    response[2] = (uint8_t) val;
-    response[1] = val >> 8;
-    response[0] = cmd;
-
     LOCK_TCPIP_CORE();
-    websocket_write(pcb, response, 3, WS_BIN_MODE);
+    websocket_write(pcb, response, 4, WS_BIN_MODE);
     UNLOCK_TCPIP_CORE();
 
-    if(data[0] == 'R') { // Restart
-        vTaskDelay(500 / portTICK_PERIOD_MS);
-        vPortEnterCritical();
-        sdk_system_restart();
-    } else if(data[0] == 'X') { // Clear Config
-        vTaskDelay(500 / portTICK_PERIOD_MS);
-        system_clear_config();
+    if(ret == OK) {
+        if(cmd == 'R' || cmd == 'C') { // Restart
+            printf("rebooting now");
+            vTaskDelay(1000 / portTICK_PERIOD_MS);
+            vPortEnterCritical();
+            sdk_system_restart();
+        } else if(cmd == 'X') { // Clear Config
+            vTaskDelay(1000 / portTICK_PERIOD_MS);
+            system_clear_config();
+        }
     }
 }
 

firmware/webdir/index.html

@@ -189,6 +189,10 @@
         return pos;
     };
 
+    DataView.prototype.getChar = function (pos) {
+        return String.fromCharCode(this.getInt8(pos));
+    };
+
     DataView.prototype.setString = function (pos, str) {
         for (var i = 0; i < str.length; i++) {
             this.setInt8(pos++, str.charCodeAt(i));
@@ -366,11 +370,11 @@
             frame.set(new Uint8Array(slice), 12);
             receive_chunk_confirmation = (dv) => {
                 setTimeout(() => {
-                    resolve(i);
+                    resolve({cmd: dv.getChar(0), ret: dv.getUint8(1), val: dv.getUint16(2)});
                 }, 50);
             }
             setTimeout(() => {
-                reject(i);
+                reject({frame_error: i});
             }, 2000);
             wsWrite(frame.buffer);
         });
@@ -384,10 +388,10 @@
             headerview.setInt32(4, buf.byteLength);
             headerview.setInt32(8, hash);
             receive_chunk_confirmation = (dv) => {
-                resolve(true);
+                resolve({cmd: dv.getChar(0), ret: dv.getUint8(1), val: dv.getUint16(2)});
             }
             setTimeout(() => {
-                reject(false);
+                reject({final_error: 0});
             }, 500);
             wsWrite(frame);
         });
@@ -400,7 +404,8 @@
                 const ash = crc32(firmware_file);
                 for (var i = 0; i * chunk_size < firmware_file.byteLength; i++) {
                     update_progress("ota", i * chunk_size / firmware_file.byteLength * 100);
-                    await transmit_firmware_chunk(firmware_file, i);
+                    const reply = await transmit_firmware_chunk(firmware_file, i);
+                    console.log("reply", reply)
                 }
                 await transmit_firmware_final(firmware_file, crc32(firmware_file));
                 update_progress("ota", 100);