minify web content

.build.yml

@@ -9,17 +9,19 @@ steps:
 - name: submodules
   image: alpine/git
   commands:
-  - git submodule update --init --recursive
+  - git submodule update --init --recursive --depth 1
 
 - name: firmware
   image: docker-repo.service.intern.lab.or.it:5000/fiatlux-build-env
   depends_on: [ submodules ]
   commands:
   - export PATH=$(pwd)/modules/sdk/xtensa-lx106-elf/bin:$PATH
+  - apt update
+  - apt install -y minify
   - make firmware -j$(nproc)
 
 - name: pcb
-  image: setsoft/kicad_auto
+  image: setsoft/kicad_auto:ki6
   commands:
   - apt update
   - apt install -y make zip
@@ -62,6 +64,6 @@ steps:
     checksum:
       - sha512
       - md5
-    title: buildtest
+    title: fiatlux
   when:
     event: tag

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "modules/rtos"]
 	path = modules/rtos
-	url = https://github.com/SuperHouse/esp-open-rtos.git
+	url = https://git.neulandlabor.de/j3d1/esp-open-rtos.git
 [submodule "modules/sdk"]
 	path = modules/sdk
 	url = https://github.com/pfalcon/esp-open-sdk.git

Makefile

@@ -1,4 +1,3 @@
-
 .PHONY: firmware flash firmware_docker case pcb
 
 all: firmware case pcb
@@ -19,6 +18,11 @@ clean:
 	+@make -C firmware clean
 	+@make -C pcb clean
 
+flash_docker:
+	sh -c "docker build -t fiatlux_firmware_env docker/firmware"
+	sh -c "docker run --volume "$$(pwd)"/firmware:/app/firmware --device=/dev/ttyUSB0 fiatlux_firmware_env make -C firmware flash"
+
+
 firmware_docker:
 	sh -c "docker build -t fiatlux_firmware_env docker/firmware"
 	sh -c "docker run --volume "$$(pwd)"/firmware:/app/firmware fiatlux_firmware_env make -C firmware html all"

firmware/Makefile

@@ -1,12 +1,12 @@
 PROGRAM=fiatlux
 
-EXTRA_CFLAGS=-O3 -Ibuild/gen
+EXTRA_CFLAGS=-O3 -Ibuild/gen -DLWIP_NETIF_HOSTNAME=1
 
-EXTRA_COMPONENTS=extras/i2s_dma extras/ws2812_i2s extras/dhcpserver extras/rboot-ota extras/mbedtls extras/httpd extras/sntp extras/cpp_support
+EXTRA_COMPONENTS=extras/i2s_dma extras/ws2812_i2s extras/dhcpserver extras/rboot-ota extras/mbedtls extras/httpd extras/sntp extras/cpp_support extras/paho_mqtt_c
 
 LIBS = hal m
 
-FLASH_MODE = dio
+FLASH_MODE = qio
 
 include ../modules/rtos/common.mk
 
@@ -15,7 +15,7 @@ html: build/gen/fsdata.c
 build/gen/fsdata.c: webdir/index.html webdir/404.html webdir/css/picnic.min.css webdir/css/style.css webdir/js/smoothie_min.js
 	@echo "Generating fsdata.."
 	@mkdir -p $(dir $@)
-	@./mkwebfs.py --gzip -o $@ $^
+	@./mkwebfs.py --gzip --minify -o $@ $^
 
 test: unittest systest
 

firmware/log.cpp

@@ -0,0 +1,49 @@
+//
+// Created by jedi on 18.11.21.
+//
+
+#include "log.h"
+
+#include <espressif/esp_common.h>
+
+constexpr unsigned syslog_buffer_size = 1024;
+char syslog_buf[syslog_buffer_size];
+volatile unsigned head = 0;
+volatile unsigned streams = 0;
+
+extern "C" void syslog(const char *msg) {
+    printf("syslog> %s", msg);
+    while (char c = *msg++) {
+        syslog_buf[head++ % syslog_buffer_size] = c;
+    }
+    syslog_buf[head] = 0;
+}
+
+unsigned syslog_current_tail() {
+    if(head < syslog_buffer_size)
+        return 0;
+    return head + 1 - syslog_buffer_size;
+}
+
+unsigned syslog_data_after(unsigned local_tail) {
+    if(local_tail > head)
+        return 0;
+    return (head % syslog_buffer_size) - (local_tail % syslog_buffer_size);
+}
+
+extern "C" int syslog_copy_out(char *out, int len, unsigned local_tail) {
+    unsigned cnt = 0;
+    while (cnt < syslog_data_after(local_tail) && cnt < len) {
+        out[cnt] = syslog_buf[local_tail % syslog_buffer_size + cnt];
+        cnt++;
+    }
+    return cnt;
+}
+
+extern "C" void syslog_attach() {
+    streams++;
+}
+
+extern "C" void syslog_detach() {
+    streams--;
+}

firmware/log.h

@@ -0,0 +1,28 @@
+//
+// Created by jedi on 18.11.21.
+//
+
+#ifndef FIRMWARE_LOG_H
+#define FIRMWARE_LOG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void syslog(const char *);
+
+unsigned syslog_current_tail();
+
+unsigned syslog_data_after(unsigned);
+
+int syslog_copy_out(char *, int, unsigned);
+
+void syslog_attach();
+
+void syslog_detach();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif //FIRMWARE_LOG_H

firmware/mkwebfs.py

@@ -2,6 +2,7 @@
 import os
 import gzip
 import argparse
+import subprocess
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-o', '--output', help='Output file name', default='stdout')
@@ -9,6 +10,9 @@ parser.add_argument('-W', '--webroot', help='Output file name', default='webdir/
 parser.add_argument('--gzip', dest='gzip', action='store_true')
 parser.add_argument('--no-gzip', dest='gzip', action='store_false')
 parser.set_defaults(gzip=False)
+parser.add_argument('--minify', dest='minify', action='store_true')
+parser.add_argument('--no-minify', dest='minify', action='store_false')
+parser.set_defaults(minify=False)
 parser.add_argument('--header', dest='header', action='store_true')
 parser.add_argument('--no-header', dest='header', action='store_false')
 parser.set_defaults(header=True)
@@ -16,6 +20,31 @@ parser.add_argument('input', nargs='+', default=os.getcwd())
 args = parser.parse_args()
 
 
+def mimeFromName(name):
+    if name.endswith(".html") or name.endswith(".htm") or name.endswith(".shtml") or name.endswith(
+            ".shtm") or name.endswith(".ssi"):
+        return "text/html"
+    if name.endswith(".js"):
+        return "application/x-javascript"
+    if name.endswith(".css"):
+        return "text/css"
+    if name.endswith(".ico"):
+        return "image/x-icon"
+    if name.endswith(".gif"):
+        return "image/gif"
+    if name.endswith(".png"):
+        return "image/png"
+    if name.endswith(".jpg"):
+        return "image/jpeg"
+    if name.endswith(".bmp"):
+        return "image/bmp"
+    if name.endswith(".class"):
+        return "application/octet-stream"
+    if name.endswith(".ram"):
+        return "audio/x-pn-realaudio"
+    return "text/plain"
+
+
 def dumpBin2CHex(f, b):
     oStr = "\t"
     n = 0
@@ -41,40 +70,28 @@ for file in httpFiles:
     webPath = ("/" + file.removeprefix(args.webroot)).replace("//", "/")
     print("{} > {}".format(file, webPath))
 
+    mimeType = mimeFromName(file)
+
     if args.header:
         if ("404" in file):
             response = b'HTTP/1.0 404 File not found\r\n'
         else:
             response = b'HTTP/1.0 200 OK\r\n'
         response += b"lwIP/1.4.1 (http://savannah.nongnu.org/projects/lwip)\r\n"
-        fext = file.split('.')[-1]
-        ctype = b'Content-type: text/plain\r\n'
-        if (fext.endswith("html") or fext.endswith("htm") or fext.endswith("shtml") or fext.endswith(
-                "shtm") or fext.endswith("ssi")):
-            ctype = b'Content-type: text/html\r\n'
-        if (fext.endswith("js")):
-            ctype = b'Content-type: application/x-javascript\r\n'
-        if (fext.endswith("css")):
-            ctype = b'Content-type: text/css\r\n'
-        if (fext.endswith("ico")):
-            ctype = b'Content-type: image/x-icon\r\n'
-        if (fext.endswith("gif")):
-            ctype = b'Content-type: image/gif\r\n'
-        if (fext.endswith("png")):
-            ctype = b'Content-type: image/png\r\n'
-        if(fext.endswith("jpg")):
-            ctype = b'Content-type: image/jpeg\r\n'
-        if(fext.endswith("bmp")):
-            ctype = b'Content-type: image/bmp\r\n'
-        if(fext.endswith("class")):
-            ctype = b'Content-type: application/octet-stream\r\n'
-        if(fext.endswith("ram")):
-            ctype = b'Content-type: audio/x-pn-realaudio\r\n'
-        response += ctype
+        response += b'Content-type: ' + mimeType.encode() + b'\r\n'
 
         binFile = open(file, 'rb')
         binData = binFile.read()
         compEff = False
+        if args.minify:
+            p = subprocess.Popen(["minify", "--html-keep-document-tags", "--mime", mimeType], stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE)
+            minData = p.communicate(binData)[0]
+            if len(minData) < len(binData):
+                print("- Minify: {} -> {}".format(len(binData), len(minData)))
+                compEff = True
+                binData = minData
+
         if args.gzip:
             compData = gzip.compress(binData, 9)
             if len(compData) < len(binData):
@@ -103,7 +120,8 @@ for file in httpFiles:
         f_fsdata_c.write("};\n\n")
 
         f_fsdata_c.write("const struct fsdata_file {}[] = {{{{\n  {},\n  {}, {} + {}, sizeof({}) - {}, 1 }}}};\n\n"
-                .format(escFileFile, lastFileStruct, escFileData, escFileData, len(fnameBin), escFileData, len(fnameBin)))
+                         .format(escFileFile, lastFileStruct, escFileData, escFileData, len(fnameBin), escFileData,
+                                 len(fnameBin)))
         # TODO: The last value is 1 if args.header == True
         lastFileStruct = escFileFile
 

firmware/otaflash.py

@@ -8,6 +8,7 @@ from websocket import WebSocketTimeoutException
 
 parser = argparse.ArgumentParser(description='Update fiatlux firmware via websocket.')
 parser.add_argument("binfile")
+parser.add_argument("address")
 
 args = parser.parse_args()
 
@@ -25,7 +26,7 @@ with open(args.binfile, "rb") as f:
     try:
         ws = websocket.WebSocket()
         print("send {}".format(args.binfile))
-        ws.connect("ws://172.16.0.1")
+        ws.connect("ws://" + args.address)
         i = 0
         bytes = f.read()
         rolling = zlib.crc32(bytes)

firmware/system.c

@@ -4,6 +4,7 @@
 
 #include "system.h"
 #include "crc32.h"
+#include "log.h"
 
 #include <FreeRTOS.h>
 #include <sysparam.h>
@@ -39,7 +40,7 @@ void system_init_config() {
     uint32_t num_sectors;
     sysparam_init(base_addr, 0);
     if(sysparam_get_info(&base_addr, &num_sectors) != SYSPARAM_OK) {
-        printf("Warning: WiFi config, sysparam not initialized\n");
+        syslog("Warning: WiFi config, sysparam not initialized\n");
         num_sectors = 0x2000 / sdk_flashchip.sector_size;
         if(sysparam_create_area(base_addr, num_sectors, true) == SYSPARAM_OK) {
             sysparam_init(base_addr, 0);
@@ -106,13 +107,13 @@ enum return_code system_otaflash_verify_and_switch(uint32_t len, uint32_t hash)
     rboot_digest_image(otaflash_context.base, min(len, MAX_IMAGE_SIZE), system_otaflash_verify_chunk, &digest);
 
     if(hash != digest) {
-        printf("OTA failed to verify firmware\r\n");
+        syslog("OTA failed to verify firmware\r\n");
         return CHECKSUM_MISMATCH;
     }
 
     vPortEnterCritical();
     if(!rboot_set_current_rom(otaflash_context.slot)) {
-        printf("OTA Update failed to set new rboot slot\r\n");
+        syslog("OTA Update failed to set new rboot slot\r\n");
         vPortExitCritical();
         return RBOOT_SWITCH_FAILED;
     }

firmware/web.cpp

@@ -6,6 +6,7 @@
 #include "system.h"
 #include "lux.h"
 #include "wifi.h"
+#include "log.h"
 
 #include <cstring>
 #include <FreeRTOS.h>
@@ -36,16 +37,34 @@ void websocket_task(void *pvParameter) {
 
     size_t connstarttime = xTaskGetTickCount();
     has_changed = {true, true, true};
+    syslog_attach();
+    unsigned local_log_tail = syslog_current_tail();
 
     for (;;) {
         if(pcb == nullptr || pcb->state != ESTABLISHED) {
-            printf("Connection closed, deleting task\n");
+            syslog("Connection closed, deleting task\n");
             break;
         }
 
+        //Syslog
+        if(syslog_data_after(local_log_tail) != 0) {
+            char response[128];
+            response[0] = 'L';
+            size_t len = syslog_copy_out(&response[4], 124, local_log_tail);
+            response[1] = len;
+            ((uint16_t &) response[2]) = local_log_tail & 0xFFFF;
+            if(len < sizeof(response)) {
+                LOCK_TCPIP_CORE();
+                websocket_write(pcb, (unsigned char *) response, len + 4, WS_BIN_MODE);
+                local_log_tail += len;
+                UNLOCK_TCPIP_CORE();
+            } else
+                syslog("buffer too small -1\n");
+            vTaskDelayMs(1000);
+        }
+
         //Global Info
         if(has_changed.global) {
-            has_changed.global = false;
             timeval tv{};
             gettimeofday(&tv, nullptr);
             size_t uptime = xTaskGetTickCount() * portTICK_PERIOD_MS / 1000;
@@ -57,7 +76,7 @@ void websocket_task(void *pvParameter) {
 
             sysparam_get_string("hostname", &hostname);
             /* Generate response in JSON format */
-            char response[160];
+            char response[192];
             size_t len = snprintf(response, sizeof(response),
                                   "{\"walltime\" : \"%d\","
                                   "\"uptime\" : \"%d\","
@@ -71,22 +90,22 @@ void websocket_task(void *pvParameter) {
             if(len < sizeof(response)) {
                 LOCK_TCPIP_CORE();
                 websocket_write(pcb, (unsigned char *) response, len, WS_TEXT_MODE);
+                has_changed.global = false;
                 UNLOCK_TCPIP_CORE();
             } else
-                printf("buffer too small 1");
-            vTaskDelayMs(2000);
+                syslog("buffer too small 0\n");
+            vTaskDelayMs(1000);
         }
 
         //Connection Info
         if(has_changed.connection) {
-            has_changed.connection = false;
             timeval tv{};
             gettimeofday(&tv, nullptr);
             size_t connuptime = (xTaskGetTickCount() - connstarttime) * portTICK_PERIOD_MS / 1000;
 
             printf("conn %d: " IPSTR " <-> " IPSTR " \n", pcb->netif_idx, IP2STR(&pcb->local_ip),
                    IP2STR(&pcb->remote_ip));
-            char response[160];
+            char response[192];
             size_t len = snprintf(response, sizeof(response),
                                   "{\"connage\" : \"%d\","
                                   "\"clientip\" : \"" IPSTR "\""
@@ -94,10 +113,11 @@ void websocket_task(void *pvParameter) {
             if(len < sizeof(response)) {
                 LOCK_TCPIP_CORE();
                 websocket_write(pcb, (unsigned char *) response, len, WS_TEXT_MODE);
+                has_changed.connection = false;
                 UNLOCK_TCPIP_CORE();
             } else
-                printf("buffer too small 1");
-            vTaskDelayMs(2000);
+                syslog("buffer too small 1\n");
+            vTaskDelayMs(1000);
         }
 
         if(has_changed.wifi) {
@@ -152,10 +172,10 @@ void websocket_task(void *pvParameter) {
                     websocket_write(pcb, (unsigned char *) response, len, WS_TEXT_MODE);
                     UNLOCK_TCPIP_CORE();
                 } else
-                    printf("buffer too small 2");
+                    syslog("buffer too small 2\n");
             }
 
-            vTaskDelayMs(2000);
+            vTaskDelayMs(1000);
 
             if(opmode == STATION_MODE || opmode == STATIONAP_MODE) {
                 uint8_t hwaddr[6];
@@ -179,7 +199,7 @@ void websocket_task(void *pvParameter) {
                     websocket_write(pcb, (unsigned char *) response, len, WS_TEXT_MODE);
                     UNLOCK_TCPIP_CORE();
                 } else
-                    printf("buffer too small 3");
+                    syslog("buffer too small 3\n");
             }
 
         }
@@ -187,6 +207,8 @@ void websocket_task(void *pvParameter) {
         vTaskDelayMs(500);
     }
 
+    syslog_detach();
+
     vTaskDelete(nullptr);
 }
 
@@ -235,12 +257,14 @@ void websocket_cb(struct tcp_pcb *pcb, char *data, u16_t data_len,
             ret = OK;
             break;
         case 'D': // Disable LED
+            syslog("G\n");
             signal_led(false);
             cmd = 'G';
             ret = OK;
             val = 1;
             break;
         case 'E': // Enable LED
+            syslog("E\n");
             signal_led(true);
             cmd = 'G';
             ret = OK;

firmware/webdir/index.html

@@ -63,6 +63,16 @@
                 </div>
             </div>
         </article>
+        <article class="card">
+            <header>
+                <h3>Syslog</h3>
+            </header>
+            <div class="table">
+                <div class="row">
+                    <pre id="syslog"></pre>
+                </div>
+            </div>
+        </article>
 
     </section>
     <section id="dashboard">
@@ -181,6 +191,7 @@
 <script>
     var menu = document.getElementById("bmenub");
     var voltage = document.getElementById("out_voltage");
+    var syslog = document.getElementById("syslog");
 
     var unused_values = {};
 
@@ -209,7 +220,7 @@
         sbox = document.getElementById('status_box');
         sbox.className = "label " + cls;
         sbox.innerHTML = text;
-        console.log(text);
+        console.info(text);
     }
 
     function startPolling() {
@@ -253,6 +264,13 @@
             } else if (cmd === 'V') {
                 voltage.innerHTML = (val * 13 / 1024).toFixed(2);
                 series.append(new Date().getTime(), val);
+            } else if (cmd === 'L') {
+                var len = dv.getUint8(1);
+                var offset = dv.getUint16(2);
+                var str = "";
+                for (var i = 0; i < len; i++)
+                    str += dv.getChar(4 + i);
+                syslog.innerHTML = syslog.innerHTML.slice(0, offset) + str;
             } else
                 console.log('unknown command', cmd, val);
         }

firmware/wifi.cpp

@@ -3,6 +3,7 @@
 //
 
 #include "wifi.h"
+#include "log.h"
 
 #include <cstdlib>
 #include <cstring>
@@ -36,7 +37,7 @@ SemaphoreHandle_t wifi_available_semaphore = nullptr;
     char *wifi_ap_ip_addr = nullptr;
     sysparam_get_string("wifi_ap_ip_addr", &wifi_ap_ip_addr);
     if(!wifi_ap_ip_addr) {
-        printf("dns: no ip address\n");
+        syslog("dns: no ip address\n");
         vTaskDelete(nullptr);
     }
     ip4_addr_t server_addr;
@@ -194,7 +195,7 @@ extern "C" void wifi_task(void *pvParameters) {
         /* If the ssid and password are not valid then disable the AP interface. */
         if(!wifi_ap_ssid || strlen(wifi_ap_ssid) < 1 || strlen(wifi_ap_ssid) >= 32 ||
            !wifi_ap_password || strlen(wifi_ap_password) < 8 || strlen(wifi_ap_password) >= 64) {
-            printf("len err\n");
+            syslog("len err\n");
             wifi_ap_enable = 0;
         }
     }

modules/rtos

@@ -1 +1 @@
-Subproject commit 503e66a500419e8863998b7ea784c5e26a7a5f7c
+Subproject commit 7faa16b07ce0d606f9525a316990da5b58e61314

pcb/.gitignore

@@ -31,6 +31,8 @@ fp-info-cache
 *.wrl
 *.step
 
-*-bak
+*-backups/
 gen/
 pcb.zip
+
+report.txt

webapp/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+src/gen/
+package-lock.json