Merge pull request #178 from sheinz/feature/spiffs_update

SPIFFS: Selectable configuration of SPIFFS

examples/spiffs/Makefile

@@ -6,6 +6,8 @@ FLASH_SIZE = 32
 SPIFFS_BASE_ADDR = 0x200000
 SPIFFS_SIZE = 0x010000
 
+# SPIFFS_SINGLETON = 0  # for run-time configuration
+
 include ../../common.mk
 
 $(eval $(call make_spiffs_image,files))

examples/spiffs/spiffs_example.c

@@ -10,6 +10,14 @@
 #include "spiffs.h"
 #include "esp_spiffs.h"
 
+/**
+ * This example shows the default SPIFFS configuration when SPIFFS is
+ * configured in compile-time (SPIFFS_SINGLETON = 1).
+ *
+ * To configure SPIFFS in run-time uncomment SPIFFS_SINGLETON in the Makefile
+ * and replace the commented esp_spiffs_init in the code below.
+ *
+ */
 
 static void example_read_file_posix()
 {
@@ -76,7 +84,13 @@ static void example_fs_info()
 
 void test_task(void *pvParameters)
 {
+#if SPIFFS_SINGLETON == 1
     esp_spiffs_init();
+#else
+    // for run-time configuration when SPIFFS_SINGLETON = 0
+    esp_spiffs_init(0x200000, 0x10000);
+#endif
+
     if (esp_spiffs_mount() != SPIFFS_OK) {
         printf("Error mount SPIFFS\n");
     }

extras/spiffs/README.md

@@ -11,15 +11,29 @@ of a flash memory.
  * SPIFFS - embedded file system for NOR flash memory.
  * POSIX file operations.
  * Static files upload to ESP8266 file system within build process.
- * SPIFFS singleton configuration. Only one instance of FS on a device.
+ * SPIFFS singleton or run-time configuration. Selectable by
+`SPIFFS_SINGLETON` variable in Makefile.
 
 ## Usage
 
+### Configuration
+
+SPIFFS can be configured in two ways. As a SINGLETON with configuration 
+parameters provided at compile-time. And during run-time. The default
+configuration is a SINGLETON. The desired configuration can be selected in
+program's Makefile with variable `SPIFFS_SINGLETON = 0`.
+
+If SPIFFS is configured in runtime (SPIFFS_SINGLETON = 0) the method 
+`esp_spiffs_init` accepts two arguments: address and size. Where address 
+and size is the location of SPIFFS region in SPI flash and its size.
+
 In order to use file system in a project the following steps should be made:
  * Add SPIFFS component in a project Makefile `EXTRA_COMPONENTS = extras/spiffs`
  * Specify your flash size in the Makefile `FLASH_SIZE = 32`
  * Specify the start address of file system region on the flash memory
-`SPIFFS_BASE_ADDR = 0x200000`
+`SPIFFS_BASE_ADDR = 0x200000`. It still needed even for `SPIFFS_SINGLETON = 0`
+in order to flash SPIFFS image to the right location during `make flash`.
+If no SPIFFS image is going to be flashed this variable can be omitted.
  * If you want to upload files to a file system during flash process specify
 the directory with files `$(eval $(call make_spiffs_image,files))`
 
@@ -63,8 +77,6 @@ with SPIFFS on a device.
 
 The build process will catch any changes in files directory and rebuild the
 image each time `make` is run.
-The build process will handle SPIFFS_SIZE change and rebuild **mkspiffs**
-utility and the image.
 
 ## Example
 

extras/spiffs/component.mk

@@ -1,7 +1,28 @@
 # Component makefile for extras/spiffs
 
+# If spiffs is configured as SINGLETON it must be configured in compile time.
+SPIFFS_SINGLETON ?= 1
+
 SPIFFS_BASE_ADDR ?= 0x300000
 SPIFFS_SIZE ?= 0x100000
+SPIFFS_LOG_PAGE_SIZE ?= 256
+SPIFFS_LOG_BLOCK_SIZE ?= 8192
+
+
+spiffs_CFLAGS += -DSPIFFS_SINGLETON=$(SPIFFS_SINGLETON)
+ifeq ($(SPIFFS_SINGLETON),1)
+# Singleton configuration
+spiffs_CFLAGS += -DSPIFFS_BASE_ADDR=$(SPIFFS_BASE_ADDR)
+spiffs_CFLAGS += -DSPIFFS_SIZE=$(SPIFFS_SIZE)
+endif
+
+spiffs_CFLAGS += -DSPIFFS_LOG_PAGE_SIZE=$(SPIFFS_LOG_PAGE_SIZE)
+spiffs_CFLAGS += -DSPIFFS_LOG_BLOCK_SIZE=$(SPIFFS_LOG_BLOCK_SIZE)
+
+# Main program needs SPIFFS definitions because it includes spiffs_config.h
+PROGRAM_CFLAGS += $(spiffs_CFLAGS)
+
+spiffs_CFLAGS := $(CFLAGS) $(spiffs_CFLAGS)
 
 INC_DIRS += $(spiffs_ROOT)
 INC_DIRS += $(spiffs_ROOT)spiffs/src
@@ -10,11 +31,6 @@ INC_DIRS += $(spiffs_ROOT)spiffs/src
 spiffs_SRC_DIR = $(spiffs_ROOT)spiffs/src
 spiffs_SRC_DIR += $(spiffs_ROOT)
 
-spiffs_CFLAGS = $(CFLAGS)
-spiffs_CFLAGS += -DSPIFFS_BASE_ADDR=$(SPIFFS_BASE_ADDR)
-spiffs_CFLAGS += -DSPIFFS_SIZE=$(SPIFFS_SIZE)
-
-
 # Create an SPIFFS image of specified directory and flash it with
 # the rest of the firmware.
 #
@@ -33,20 +49,16 @@ all: $$(SPIFFS_IMAGE)
 
 clean: clean_spiffs_img clean_mkspiffs
 
-$$(SPIFFS_IMAGE): $$(MKSPIFFS) $$(SPIFFS_FILE_LIST)
+$$(SPIFFS_IMAGE): $$(MKSPIFFS) $$(SPIFFS_FILE_LIST) Makefile
-	$$< $(1) $$@
+	$$< -D $(1) -f $$@ -s $(SPIFFS_SIZE) -p $(SPIFFS_LOG_PAGE_SIZE) \
+		-b $(SPIFFS_LOG_BLOCK_SIZE)
 
 # Rebuild SPIFFS if Makefile is changed, where SPIFF_SIZE is defined
 $$(spiffs_ROOT)spiffs_config.h: Makefile
 	$$(Q) touch $$@
 
-$$(MKSPIFFS)_MAKE:
+$$(MKSPIFFS):
-	$$(MAKE) -C $$(MKSPIFFS_DIR) SPIFFS_SIZE=$(SPIFFS_SIZE)
+	$$(MAKE) -C $$(MKSPIFFS_DIR)
-
-# if SPIFFS_SIZE in Makefile is changed rebuild mkspiffs
-$$(MKSPIFFS): Makefile
-	$$(MAKE) -C $$(MKSPIFFS_DIR) clean
-	$$(MAKE) -C $$(MKSPIFFS_DIR) SPIFFS_SIZE=$(SPIFFS_SIZE)
 
 clean_spiffs_img:
 	$$(Q) rm -f $$(SPIFFS_IMAGE)
@@ -54,11 +66,6 @@ clean_spiffs_img:
 clean_mkspiffs:
 	$$(Q) $$(MAKE) -C $$(MKSPIFFS_DIR) clean
 
-# run make for mkspiffs always
-all: $$(MKSPIFFS)_MAKE
-
-.PHONY: $$(MKSPIFFS)_MAKE
-
 SPIFFS_ESPTOOL_ARGS = $(SPIFFS_BASE_ADDR) $$(SPIFFS_IMAGE)
 endef
 

extras/spiffs/esp_spiffs.c

@@ -20,6 +20,7 @@ typedef struct {
     uint32_t size;
 } fs_buf_t;
 
+static spiffs_config config = {0};
 static fs_buf_t work_buf = {0};
 static fs_buf_t fds_buf = {0};
 static fs_buf_t cache_buf = {0};
@@ -63,15 +64,36 @@ static s32_t esp_spiffs_erase(u32_t addr, u32_t size)
     return SPIFFS_OK;
 }
 
+#if SPIFFS_SINGLETON == 1
 void esp_spiffs_init()
 {
-    work_buf.size = 2 * SPIFFS_CFG_LOG_PAGE_SZ();
+#else
+void esp_spiffs_init(uint32_t addr, uint32_t size)
+{
+    config.phys_addr = addr;
+    config.phys_size = size;
+
+    config.phys_erase_block = SPIFFS_ESP_ERASE_SIZE;
+    config.log_page_size = SPIFFS_LOG_PAGE_SIZE;
+    config.log_block_size = SPIFFS_LOG_BLOCK_SIZE;
+
+    // Initialize fs.cfg so the following helper functions work correctly
+    memcpy(&fs.cfg, &config, sizeof(spiffs_config));
+#endif
+    work_buf.size = 2 * SPIFFS_LOG_PAGE_SIZE;
     fds_buf.size = SPIFFS_buffer_bytes_for_filedescs(&fs, ESP_SPIFFS_FD_NUMBER);
     cache_buf.size= SPIFFS_buffer_bytes_for_cache(&fs, ESP_SPIFFS_CACHE_PAGES);
 
     work_buf.buf = malloc(work_buf.size);
     fds_buf.buf = malloc(fds_buf.size);
     cache_buf.buf = malloc(cache_buf.size);
+
+    config.hal_read_f = esp_spiffs_read;
+    config.hal_write_f = esp_spiffs_write;
+    config.hal_erase_f = esp_spiffs_erase;
+
+    config.fh_ix_offset = 3;
+
 }
 
 void esp_spiffs_deinit()
@@ -88,15 +110,6 @@ void esp_spiffs_deinit()
 
 int32_t esp_spiffs_mount()
 {
-    spiffs_config config = {0};
-
-    config.hal_read_f = esp_spiffs_read;
-    config.hal_write_f = esp_spiffs_write;
-    config.hal_erase_f = esp_spiffs_erase;
-
-    config.fh_ix_offset = 3;
-
-    printf("SPIFFS size: %d\n", SPIFFS_SIZE);
     printf("SPIFFS memory, work_buf_size=%d, fds_buf_size=%d, cache_buf_size=%d\n",
             work_buf.size, fds_buf.size, cache_buf.size);
 
@@ -154,7 +167,7 @@ int _open_r(struct _reent *r, const char *pathname, int flags, int mode)
     if (flags & O_TRUNC)    spiffs_flags |= SPIFFS_TRUNC;
     if (flags & O_RDONLY)   spiffs_flags |= SPIFFS_RDONLY;
     if (flags & O_WRONLY)   spiffs_flags |= SPIFFS_WRONLY;
-    if (flags & O_EXCL)     spiffs_flags |= SPIFFS_EXCL; 
+    if (flags & O_EXCL)     spiffs_flags |= SPIFFS_EXCL;
     /* if (flags & O_DIRECT)   spiffs_flags |= SPIFFS_DIRECT; no support in newlib */
 
     return SPIFFS_open(&fs, pathname, spiffs_flags, mode);

extras/spiffs/esp_spiffs.h

@@ -12,12 +12,25 @@
 
 extern spiffs fs;
 
+#if SPIFFS_SINGLETON == 1
 /**
  * Prepare for SPIFFS mount.
  *
  * The function allocates all the necessary buffers.
  */
 void esp_spiffs_init();
+#else
+/**
+ * Prepare for SPIFFS mount.
+ *
+ * The function allocates all the necessary buffers.
+ *
+ * @param addr Base address for spiffs in flash memory.
+ * @param size File sistem size.
+ */
+void esp_spiffs_init(uint32_t addr, uint32_t size);
+#endif
+
 
 /**
  * Free all memory buffers that were used by SPIFFS.

extras/spiffs/mkspiffs/Makefile

@@ -1,16 +1,3 @@
-# Check if SPIFFS_SIZE defined only if not cleaning
-ifneq ($(MAKECMDGOALS),clean)
-ifndef SPIFFS_SIZE
-define ERROR_MSG
-Variable SPIFFS_SIZE is not defined.
-Cannot build mkspiffs without SPIFFS_SIZE.
-Please specify it in your application Makefile.
-
-endef
-$(error $(ERROR_MSG))	
-endif
-endif
-
 # explicitly use gcc as in xtensa build environment it might be set to
 # cross compiler
 CC = gcc
@@ -27,8 +14,7 @@ OBJECTS := $(SOURCES:.c=.o)
 VPATH = ../spiffs/src
 
 CFLAGS += -I..
-CFLAGS += -DSPIFFS_BASE_ADDR=0   # for image base addr is start of the image
+CFLAGS += -DSPIFFS_SINGLETON=0
-CFLAGS += -DSPIFFS_SIZE=$(SPIFFS_SIZE)
 
 all: mkspiffs
 

extras/spiffs/mkspiffs/README.md

@@ -14,21 +14,22 @@ $(eval $(call make_spiffs_image,files))
 
 where *files* is the directory with files that should go into SPIFFS image.
 
-Or you can build mkspiffs manually with:
+mkspiffs can be built separately. Simply run `make` in the mkspiffs directory.
+
+To manually generate SPIFFS image from a directory SPIFFS configuration must be
+provided as command line arguments.
+
+Arguments:
+ * -D Directory with files that will be put in SPIFFS image.
+ * -f SPIFFS image file name.
+ * -s SPIFFS size.
+ * -p Logical page size.
+ * -b Logical block size.
+
+All arguments are mandatory.
+
+For example:
 
 ```
-make SPIFFS_SIZE=0x100000
+mkspiffs -D ./my_files -f spiffs.img -s 0x10000 -p 256 -b 8192
-```
-
-mkspiffs cannot be built without specifying SPIFFS size because it uses the
-same SPIFFS sources as the firmware. And for the firmware SPIFFS size is
-compile time defined.
-
-Please note that if you change SPIFFS_SIZE you need to rebuild mkspiffs.
-The easiest way is to run `make clean` for you project.
-
-To manually generate SPIFFS image from directory, run:
-
-```
-mkspiffs DIRECTORY IMAGE_NAME
 ```

extras/spiffs/mkspiffs/mkspiffs.c

@@ -31,6 +31,7 @@
 #include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
+#include <getopt.h>
 #include "spiffs_config.h"
 #include "../spiffs/src/spiffs.h"
 
@@ -40,15 +41,23 @@ static void *work_buf = 0;
 static void *fds_buf = 0;
 static void *cache_buf = 0;
 
+typedef struct {
+    uint32_t fs_size;
+    uint32_t log_page_size;
+    uint32_t log_block_size;
+} fs_config_t;
+
 static void print_usage(const char *prog_name, const char *error_msg)
 {
     if (error_msg) {
         printf("Error: %s\n", error_msg);
     }
     printf("Usage: ");
-    printf("\t%s DIRECTORY IMAGE_NAME\n\n", prog_name);
+    printf("%s [-D directory] [-f image-name] [-s size]\n", prog_name);
+    printf("\t[-p page-size] [-b block-size]\n\n");
     printf("Example:\n");
-    printf("\t%s ./my_files spiffs.img\n\n", prog_name);
+    printf("\t%s -D ./my_files -f spiffs.img -s 0x10000 -p 256 -b 8192\n\n", 
+            prog_name);
 }
 
 static s32_t _read_data(u32_t addr, u32_t size, u8_t *dst)
@@ -74,21 +83,30 @@ static s32_t _erase_data(u32_t addr, u32_t size)
     return SPIFFS_OK;
 }
 
-static bool init_spiffs(bool allocate_mem)
+static bool init_spiffs(bool allocate_mem, const fs_config_t *fs_config)
 {
     spiffs_config config = {0};
-    printf("Initializing SPIFFS, size=%d\n", SPIFFS_SIZE);
+    printf("Initializing SPIFFS, size=%d\n", fs_config->fs_size);
 
     config.hal_read_f = _read_data;
     config.hal_write_f = _write_data;
     config.hal_erase_f = _erase_data;
 
-    int workBufSize = 2 * SPIFFS_CFG_LOG_PAGE_SZ();
+    config.phys_addr = 0;
+    config.phys_size = fs_config->fs_size;
+    config.log_page_size = fs_config->log_page_size;
+    config.log_block_size = fs_config->log_block_size;
+    config.phys_erase_block = SPIFFS_ESP_ERASE_SIZE;
+
+    // initialize fs.cfg so the following helper functions work correctly
+    memcpy(&fs.cfg, &config, sizeof(spiffs_config));
+
+    int workBufSize = 2 * fs_config->log_page_size;
     int fdsBufSize = SPIFFS_buffer_bytes_for_filedescs(&fs, 5);
     int cacheBufSize = SPIFFS_buffer_bytes_for_cache(&fs, 5);
 
     if (allocate_mem) {
-        image = malloc(SPIFFS_SIZE);
+        image = malloc(fs_config->fs_size);
         work_buf = malloc(workBufSize);
         fds_buf = malloc(fdsBufSize);
         cache_buf = malloc(cacheBufSize);
@@ -107,22 +125,18 @@ static bool format_spiffs()
     SPIFFS_unmount(&fs);
 
     if (SPIFFS_format(&fs) == SPIFFS_OK) {
-        printf("Format complete\n"); 
+        printf("Format complete\n");
     } else {
         printf("Failed to format SPIFFS\n");
         return false;
     }
 
-    if (!init_spiffs(false)) {
-        printf("Failed to mount SPIFFS\n");
-        return false;
-    }
     return true;
 }
 
 static void spiffs_free()
 {
-    free(image); 
+    free(image);
     image = NULL;
 
     free(work_buf);
@@ -141,19 +155,19 @@ static bool process_file(const char *src_file, const char *dst_file)
     const int buf_size = 256;
     uint8_t buf[buf_size];
     int data_len;
-    
+
     fd = open(src_file, O_RDONLY);
     if (fd < 0) {
         printf("Error openning file: %s\n", src_file);
     }
-    
+
-    spiffs_file out_fd = SPIFFS_open(&fs, dst_file, 
+    spiffs_file out_fd = SPIFFS_open(&fs, dst_file,
             SPIFFS_O_CREAT | SPIFFS_O_WRONLY, 0);
     while ((data_len = read(fd, buf, buf_size)) != 0) {
         if (SPIFFS_write(&fs, out_fd, buf, data_len) != data_len) {
             printf("Error writing to SPIFFS file\n");
             break;
-        }       
+        }
     }
     SPIFFS_close(&fs, out_fd);
     close(fd);
@@ -169,7 +183,7 @@ static bool process_directory(const char *direcotry)
     dp = opendir(direcotry);
     if (dp != NULL) {
         while ((ep = readdir(dp)) != 0) {
-            if (!strcmp(ep->d_name, ".") || 
+            if (!strcmp(ep->d_name, ".") ||
                 !strcmp(ep->d_name, "..")) {
                 continue;
             }
@@ -179,7 +193,7 @@ static bool process_directory(const char *direcotry)
             sprintf(path, "%s/%s", direcotry, ep->d_name);
             printf("Processing file %s\n", path);
             if (!process_file(path, ep->d_name)) {
-                printf("Error processing file\n");   
+                printf("Error processing file\n");
                 break;
             }
         }
@@ -190,10 +204,10 @@ static bool process_directory(const char *direcotry)
     return true;
 }
 
-static bool write_image(const char *out_file)
+static bool write_image(const char *out_file, const fs_config_t *fs_config)
 {
     int fd;
-    int size = SPIFFS_SIZE;
+    int size = fs_config->fs_size;
     uint8_t *p = (uint8_t*)image;
     fd = open(out_file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
     if (fd < 0) {
@@ -204,9 +218,9 @@ static bool write_image(const char *out_file)
     printf("Writing image to file: %s\n", out_file);
 
     while (size != 0) {
-        write(fd, p, SPIFFS_CFG_LOG_PAGE_SZ());
+        write(fd, p, fs_config->log_page_size);
-        p += SPIFFS_CFG_LOG_PAGE_SZ();
+        p += fs_config->log_page_size;
-        size -= SPIFFS_CFG_LOG_PAGE_SZ();
+        size -= fs_config->log_page_size;
     }
 
     close(fd);
@@ -216,26 +230,59 @@ static bool write_image(const char *out_file)
 int main(int argc, char *argv[])
 {
     int result = 0;
-    
+    int option = 0;
-    if (argc != 3) {
+    fs_config_t fs_config = {0};
+    char *image_file_name = 0;
+    char *directory = 0;
+
+    while ((option = getopt(argc, argv, "D:f:s:p:b:e:")) != -1) {
+        switch (option) {
+            case 'D':  // directory
+                directory = optarg;
+                break;
+            case 'f':  // image file name
+                image_file_name = optarg;
+                break;
+            case 's':  // file system size
+                fs_config.fs_size = (uint32_t)strtol(optarg, NULL, 0);
+                break;
+            case 'p':  // logical page size
+                fs_config.log_page_size = (uint32_t)strtol(optarg, NULL, 0);
+                break;
+            case 'b':  // logical block size
+                fs_config.log_block_size = (uint32_t)strtol(optarg, NULL, 0);
+                break;
+            default:
+                print_usage(argv[0], NULL);
+                return -1;
+        }
+    }
+
+    if (!image_file_name || !directory || !fs_config.fs_size
+            || !fs_config.log_page_size || !fs_config.log_block_size) {
         print_usage(argv[0], NULL);
         return -1;
     }
 
-    init_spiffs(/*allocate_mem=*/true);
+    init_spiffs(/*allocate_mem=*/true, &fs_config);
 
     if (format_spiffs()) {
-        if (process_directory(argv[1])) {
+        if (init_spiffs(/*allocate_mem=*/false, &fs_config)) {
-            if (!write_image(argv[2])) {
+            if (process_directory(directory)) {
-                printf("Error writing image\n");
+                SPIFFS_unmount(&fs);
-            }       
+                if (!write_image(image_file_name, &fs_config)) {
+                    printf("Error writing image\n");
+                }
+            } else {
+                printf("Error processing direcotry\n");
+            }
         } else {
-            printf("Error processing direcotry\n");
+            printf("Failed to mount SPIFFS\n");
         }
     } else {
         printf("Error formating spiffs\n");
     }
-      
+
     spiffs_free();
     return result;
 }

extras/spiffs/spiffs_config.h

@@ -158,23 +158,26 @@ typedef unsigned char u8_t;
 #define SPIFFS_SINGLETON 1
 #endif
 
+// ESP8266 supports only sector erase, which is 4096 bytes
+#define SPIFFS_ESP_ERASE_SIZE       (4096)
+
 #if SPIFFS_SINGLETON
 // Instead of giving parameters in config struct, singleton build must
 // give parameters in defines below.
 #ifndef SPIFFS_CFG_PHYS_SZ
-#define SPIFFS_CFG_PHYS_SZ(ignore)        (SPIFFS_SIZE)
+#define SPIFFS_CFG_PHYS_SZ(ignore)          (SPIFFS_SIZE)
 #endif
 #ifndef SPIFFS_CFG_PHYS_ERASE_SZ
-#define SPIFFS_CFG_PHYS_ERASE_SZ(ignore)  (4*1024)
+#define SPIFFS_CFG_PHYS_ERASE_SZ(ignore)    (SPIFFS_ESP_ERASE_SIZE)
 #endif
 #ifndef SPIFFS_CFG_PHYS_ADDR
-#define SPIFFS_CFG_PHYS_ADDR(ignore)      (SPIFFS_BASE_ADDR)
+#define SPIFFS_CFG_PHYS_ADDR(ignore)        (SPIFFS_BASE_ADDR)
 #endif
 #ifndef SPIFFS_CFG_LOG_PAGE_SZ
-#define SPIFFS_CFG_LOG_PAGE_SZ(ignore)    (256)
+#define SPIFFS_CFG_LOG_PAGE_SZ(ignore)      (SPIFFS_LOG_PAGE_SIZE)
 #endif
 #ifndef SPIFFS_CFG_LOG_BLOCK_SZ
-#define SPIFFS_CFG_LOG_BLOCK_SZ(ignore)   (8*1024)
+#define SPIFFS_CFG_LOG_BLOCK_SZ(ignore)     (SPIFFS_LOG_BLOCK_SIZE)
 #endif
 #endif