Almost functional OTA support

ota_basic example can receive new image via TCP.

However - writing to flash with interrupts disabled causes data loss,
and the TCP flow is very slow to recover. Linux sender quickly ramps up
RTT timer to very long retry intervals, crippling performance &
throughput.

Running the update without the flash writes causes the data to be
received quickly, so this is definitely an issue with the time taken for
the erase cycle.

Progress towards #10

common.mk

@@ -230,6 +230,7 @@ $$($(1)_OBJ_DIR)%.o: $$($(1)_REAL_ROOT)%.s $$($(1)_MAKEFILE) $(wildcard $(ROOT)*
 # for missing explicitly named source files
 $$($(1)_AR): $$($(1)_OBJ_FILES) $$($(1)_SRC_FILES)
 	$(vecho) "AR $$@"
+	$(Q) mkdir -p $$(dir $$@)
 	$(Q) $(AR) cru $$@ $$^
 
 COMPONENT_ARS += $$($(1)_AR)

examples/ota_basic/Makefile

@@ -0,0 +1,5 @@
+PROGRAM=ota_basic
+OTA=1
+EXTRA_COMPONENTS=extras/rboot-ota
+include ../../common.mk
+

examples/ota_basic/ota_basic.c

@@ -0,0 +1,90 @@
+/* A very simple OTA example
+ *
+ * Binds a TCP socket, reads an image from it and then flashes live.
+ *
+ * This lets you flash from the command line via netcat.
+ *
+ * NOT SUITABLE TO PUT ON THE INTERNET OR INTO A PRODUCTION ENVIRONMENT!!!!
+ */
+#include "espressif/esp_common.h"
+#include "espressif/sdk_private.h"
+#include "FreeRTOS.h"
+#include "task.h"
+#include "esp8266.h"
+#include "rboot-ota.h"
+
+#include "lwip/err.h"
+#include "lwip/sockets.h"
+#include "lwip/sys.h"
+#include "lwip/netdb.h"
+#include "lwip/dns.h"
+
+#define FWPORT 12550
+
+#if !defined(WIFI_SSID) || !defined(WIFI_PASS)
+#error "Please define macros WIFI_SSID & WIFI_PASS (here, or better in a local.h file at root level or in program dir."
+#endif
+
+void simpleOTATask(void *pvParameters)
+{
+    printf("Listening for firmware on port %d...\r\n", FWPORT);
+    int s = socket(AF_INET, SOCK_STREAM, 0);
+    if(s < 0) {
+        printf("... Failed to allocate socket.\r\n");
+        return;
+    }
+
+    struct sockaddr_in s_addr = {
+        .sin_family = AF_INET,
+        .sin_addr = {
+            .s_addr = INADDR_ANY,
+        },
+        .sin_port = htons(FWPORT),
+    };
+    if(bind(s, (struct sockaddr *) &s_addr, sizeof(s_addr)) < 0)
+    {
+        printf("... Failed to bind.\r\n");
+        return;
+    }
+
+    if(listen(s, 0) == -1) {
+        printf("... Failed to listen.\r\n");
+        return;
+    }
+
+    int client;
+    while((client = accept(s, NULL, NULL)) != 0)
+    {
+        printf("Got new socket. Trying OTA update...\r\n");
+
+        int slot = rboot_ota_update(client, -1, false);
+        close(client);
+        if(slot < 0) {
+            printf("OTA update failed. :(.\r\n");
+            continue;
+        }
+        printf("OTA succeeded at slot %d!\r\n", slot);
+        //rboot_set_current_rom(slot);
+        //sdk_system_restart();
+    }
+    printf("Failed to accept.\r\n");
+    close(s);
+    return;
+}
+
+void user_init(void)
+{
+    sdk_uart_div_modify(0, UART_CLK_FREQ / 115200);
+
+    printf("OTA Basic demo. Currently running on slot %d / %d.\r\n",
+           rboot_get_current_rom(), RBOOT_MAX_ROMS);
+
+    struct sdk_station_config config = {
+        .ssid = WIFI_SSID,
+        .password = WIFI_PASS,
+    };
+    sdk_wifi_set_opmode(STATION_MODE);
+    sdk_wifi_station_set_config(&config);
+
+    xTaskCreate(simpleOTATask, (signed char *)"simpleOTATask", 512, NULL, 2, NULL);
+}

examples/ota_basic/ota_basic.def

@@ -0,0 +1,9 @@
+cpu xtensa
+
+# Show up to this many raw bytes of code/data
+show bytes 4
+
+# ELF file
+# See example.def if you want to load raw binaries instead
+load build/ota_basic.out elf
+

examples/ota_basic/test.c

@@ -0,0 +1,7 @@
+extern void wDev_ProcessFiq(void);
+
+void call_wdev(void)
+{
+    wDev_ProcessFiq();
+}
+

extras/rboot-ota/component.mk

@@ -0,0 +1,9 @@
+# Component makefile for extras/rboot-ota
+
+INC_DIRS += $(ROOT)extras/rboot-ota
+
+# args for passing into compile rule generation
+extras/rboot-ota_INC_DIR =  $(ROOT)extras/rboot-ota
+extras/rboot-ota_SRC_DIR =  $(ROOT)extras/rboot-ota
+
+$(eval $(call component_compile_rules,extras/rboot-ota))

extras/rboot-ota/rboot-config.h

@@ -0,0 +1,14 @@
+#ifndef _RBOOT_CONFIG_H
+/* rboot configuration parameters.
+
+   Must match the config in the compiler bootloader rboot.h. Values below are the rboot.h defaults.
+
+   Override rboot parameters by editing this file, or (much better
+   alternative) copy rboot-config.h to your program directory or
+   program/include/ directory, then edit it to override these values.
+*/
+
+#define RBOOT_MAX_ROMS 4
+//#define RBOOT_CONFIG_CHECKSUM
+
+#endif

extras/rboot-ota/rboot-ota.c

@@ -0,0 +1,203 @@
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <espressif/spi_flash.h>
+#include <espressif/esp_system.h>
+
+#include <lwip/sockets.h>
+
+#include <FreeRTOS.h>
+#include <task.h>
+
+#define SECTOR_SIZE 0x1000
+#define BOOT_CONFIG_SECTOR 1
+
+#include "rboot-ota.h"
+
+#define ROM_MAGIC_OLD 0xe9
+#define ROM_MAGIC_NEW 0xea
+
+
+// get the rboot config
+rboot_config_t rboot_get_config() {
+	rboot_config_t conf;
+	sdk_spi_flash_read(BOOT_CONFIG_SECTOR * SECTOR_SIZE, (uint32_t*)&conf, sizeof(rboot_config_t));
+	return conf;
+}
+
+// write the rboot config
+// preserves contents of rest of sector, so rest
+// of sector can be used to store user data
+// updates checksum automatically, if enabled
+bool rboot_set_config(rboot_config_t *conf) {
+	uint8_t *buffer;
+#ifdef RBOOT_CONFIG_CHKSUM
+	uint8_t chksum;
+	uint8_t *ptr;
+#endif
+
+	buffer = (uint8_t*)malloc(SECTOR_SIZE);
+	if (!buffer) {
+		printf("Failed to allocate sector buffer\r\n");
+		return false;
+	}
+
+#ifdef BOOT_CONFIG_CHKSUM
+	chksum = CHKSUM_INIT;
+	for (ptr = (uint8_t*)conf; ptr < &conf->chksum; ptr++) {
+		chksum ^= *ptr;
+	}
+	conf->chksum = chksum;
+#endif
+
+	sdk_spi_flash_read(BOOT_CONFIG_SECTOR * SECTOR_SIZE, (uint32_t*)buffer, SECTOR_SIZE);
+	memcpy(buffer, conf, sizeof(rboot_config_t));
+        vPortEnterCritical();
+	sdk_spi_flash_erase_sector(BOOT_CONFIG_SECTOR);
+        vPortExitCritical();
+        taskYIELD();
+        vPortEnterCritical();
+	sdk_spi_flash_write(BOOT_CONFIG_SECTOR * SECTOR_SIZE, (uint32_t*)buffer, SECTOR_SIZE);
+        vPortExitCritical();
+	free(buffer);
+	return true;
+}
+
+// get current boot rom
+uint8_t rboot_get_current_rom() {
+	rboot_config_t conf;
+	conf = rboot_get_config();
+	return conf.current_rom;
+}
+
+// set current boot rom
+bool rboot_set_current_rom(uint8_t rom) {
+	rboot_config_t conf;
+	conf = rboot_get_config();
+	if (rom >= conf.count) return false;
+	conf.current_rom = rom;
+	return rboot_set_config(&conf);
+}
+
+int rboot_ota_update(int fd, int slot, bool reboot_now)
+{
+    rboot_config_t conf;
+    conf = rboot_get_config();
+
+    if(slot == -1) {
+        slot = (conf.current_rom + 1) % RBOOT_MAX_ROMS;
+    }
+
+    size_t sector = conf.roms[slot] / SECTOR_SIZE;
+    uint8_t *sector_buf = (uint8_t *)malloc(SECTOR_SIZE);
+    if(!sector_buf) {
+        printf("Failed to malloc sector buffer\r\n");
+        return 0;
+    }
+
+    printf("New image going into sector %d @ 0x%lx\r\n", slot, conf.roms[slot]);
+
+    /* Read bootloader header */
+    int r = read(fd, sector_buf, 8);
+    if(r != 8 || (sector_buf[0] != ROM_MAGIC_OLD && sector_buf[0] != ROM_MAGIC_NEW)) {
+        printf("Failed to read ESP bootloader header r=%d magic=%d.\r\n", r, sector_buf[0]);
+        slot = -1;
+        goto cleanup;
+    }
+    /* if we saw a v1.2 header, we can expect a v1.1 header after the first section */
+    bool in_new_header = (sector_buf[0] == ROM_MAGIC_NEW);
+
+    int remaining_sections = sector_buf[1];
+    size_t offs = 8;
+    size_t total_read = 8;
+    size_t left_section = 0; /* Number of bytes left in this section */
+
+    while(remaining_sections > 0 || left_section > 0)
+    {
+        if(left_section == 0) {
+            /* No more bytes in this section, read the next section header */
+
+            if(offs + (in_new_header ? 16 : 8) > SECTOR_SIZE) {
+                printf("PANIC: reading section header overflows sector boundary. FIXME.\r\n");
+                slot = -1;
+                goto cleanup;
+            }
+
+            if(in_new_header && total_read > 8)
+            {
+                /* expecting an "old" style 8 byte image header here, following irom0 */
+                int r = read(fd, sector_buf+offs, 8);
+                if(r != 8 || sector_buf[offs] != ROM_MAGIC_OLD) {
+                    printf("Failed to read second flash header r=%d magic=0x%02x.\r\n", r, sector_buf[offs]);
+                    slot = -1;
+                    goto cleanup;
+                }
+                /* treat this number as the reliable number of remaining sections */
+                remaining_sections = sector_buf[offs+1];
+                in_new_header = false;
+            }
+
+            int r = read(fd, sector_buf+offs, 8);
+            if(r != 8) {
+                printf("Failed to read section header.\r\n");
+                slot = -1;
+                goto cleanup;
+            }
+            offs += 8;
+            total_read += 8;
+            left_section = *((uint32_t *)(sector_buf+offs-4));
+
+            /* account for padding from the reported section size out to the alignment barrier */
+            size_t section_align = in_new_header ? 16 : 4;
+            left_section = (left_section+section_align-1) & ~(section_align-1);
+
+            remaining_sections--;
+            printf("New section @ 0x%x length 0x%x align 0x%x remaining 0x%x\r\n", total_read, left_section, section_align, remaining_sections);
+        }
+
+        size_t bytes_to_read = left_section > (SECTOR_SIZE-offs) ? (SECTOR_SIZE-offs) : left_section;
+        int r = read(fd, sector_buf+offs, bytes_to_read);
+        if(r < 0) {
+            printf("Failed to read from fd\r\n");
+            slot = -1;
+            goto cleanup;
+        }
+        if(r == 0) {
+            printf("EOF before end of image remaining_sections=%d this_section=%d\r\n",
+                   remaining_sections, left_section);
+            slot = -1;
+            goto cleanup;
+        }
+        offs += r;
+        total_read += r;
+        left_section -= r;
+
+        if(offs == SECTOR_SIZE || (left_section == 0 && remaining_sections == 0)) {
+            /* sector buffer is full, or we're at the very end, so write sector to flash */
+            printf("Sector buffer full. Erasing sector 0x%02x\r\n", sector);
+            sdk_spi_flash_erase_sector(sector);
+            printf("Erase done.\r\n");
+            //sdk_spi_flash_write(sector * SECTOR_SIZE, (uint32_t*)sector_buf, SECTOR_SIZE);
+            printf("Flash done.\r\n");
+            offs = 0;
+            sector++;
+        }
+    }
+
+    /* Done reading image from fd and writing it out */
+    if(reboot_now)
+    {
+        close(fd);
+        conf.current_rom = slot;
+        rboot_set_config(&conf);
+        sdk_system_restart();
+    }
+
+ cleanup:
+    free(sector_buf);
+    return slot;
+}
+

extras/rboot-ota/rboot-ota.h

@@ -0,0 +1,62 @@
+#ifndef __RBOOT_OTA_H__
+#define __RBOOTOTA_H__
+/* rboot-ota client API
+ *
+ * Ported from https://github.com/raburton/esp8266/ to esp-open-rtos
+ *
+ * BSD Licensed as per the file LICENSE in the top-level directory.
+ * Copyright (c) 2015 Richard A Burton & SuperHouse Pty Ltd
+ */
+#include <stdint.h>
+#include <rboot-config.h>
+
+/* rboot config block structure (stored in flash offset 0x1000)
+ *
+ * Structure taken from rboot.h revision a4724ede22
+ * The version of rboot you're using has to match this structure
+ */
+typedef struct {
+        uint8_t magic;               // our magic
+        uint8_t version;             // config struct version
+        uint8_t mode;                        // boot loader mode
+        uint8_t current_rom;         // currently selected rom
+        uint8_t gpio_rom;            // rom to use for gpio boot
+        uint8_t count;               // number of roms in use
+        uint8_t unused[2];           // padding
+        uint32_t roms[RBOOT_MAX_ROMS]; // flash addresses of the roms
+#ifdef RBOOT_CONFIG_CHKSUM
+        uint8_t chksum;              // config chksum
+#endif
+} rboot_config_t;
+
+
+// timeout for the initial connect (in ms)
+#define OTA_CONNECT_TIMEOUT  10000
+
+// timeout for the download and flash to complete (in ms), once connected
+#define OTA_DOWNLOAD_TIMEOUT 20000
+
+#define UPGRADE_FLAG_IDLE		0x00
+#define UPGRADE_FLAG_START		0x01
+#define UPGRADE_FLAG_FINISH		0x02
+
+#define FLASH_BY_ADDR 0xff
+
+/* Perform an OTA update.
+ *
+ * * 'fd' is the file descriptor to read the OTA image from.
+ * * 'slot' is the slot to update, or -1 to automatically update next slot.
+ * * 'reboot_now' means to reboot to the new slot immediately
+ *   (if true, function won't return if successful).
+ *
+ * Returns '0' if the update fails. Returns the newly loaded slot if
+ * reboot_now is false, and the update succeeds so the next restart
+ * will hit the new image.
+ */
+int rboot_ota_update(int fd, int slot, bool reboot_now);
+rboot_config_t rboot_get_config();
+bool rboot_set_config(rboot_config_t *conf);
+uint8_t rboot_get_current_rom();
+bool rboot_set_current_rom(uint8_t rom);
+
+#endif

extras/rboot-ota/readme.txt

@@ -0,0 +1,49 @@
+rBoot - User API and OTA support for rBoot on the ESP8266
+---------------------------------------------------------
+by Richard A Burton, richardaburton@gmail.com
+http://richard.burtons.org/
+
+See rBoot readme.txt for how to create suitable rom images to update OTA.
+
+This provides a couple of simple APIs for getting rBoot config and one for over
+the air updates (OTA):
+
+  rboot_config rboot_get_config();
+    Returns rboot_config (defined in rboot.h) allowing you to modify any values
+    in it, including the rom layout.
+
+  bool rboot_set_config(rboot_config *conf);
+    Saves the rboot_config structure back to sector 2 of the flash, while
+    maintaining the contents of the rest of the sector. You can use the rest of
+    this sector for your app settings, as long as protect this structure when
+    you do so.
+
+  uint8 rboot_get_current_rom();
+    Get the currently selected boot rom (the currently running rom, as long as
+	you haven't changed it since boot).
+
+  bool rboot_set_current_rom(uint8 rom);
+    Set the current boot rom, which will be used when next restarted.
+
+  bool rboot_ota_start(rboot_ota *ota);
+    Starts an OTA. Pass it an rboot_ota structure with appropriate options. This
+    function works very much like the SDK libupgrade code you may already be
+	using, very few changes will be needed to switch to this.
+    
+    typedef struct {
+        uint8 ip[4];
+        uint16 port;
+        uint8 *request;
+        uint8 rom_slot;
+        uint32 rom_addr;
+        ota_callback callback;
+    } rboot_ota;
+    
+    - ip is the ip of the OTA http server.
+    - port is the server port (e.g. 80).
+    - request is the http request to send.
+    - rom_slot is the rom slot to write to (numbered from zero), or set to
+      FLASH_BY_ADDR to flash by address instead of by rom slot.
+    - rom_addr is the flash address to write to when using FLASH_BY_ADDR,
+      otherwise it is ignored.
+    - call back is the user code function to call on completion of OTA.