Continue boot on bad PSRAM (#1558)

* add sensor status, show special index/setup page in case of a critical error

* continue booting on PSRAM error

* Update main.cpp

added log messages

* init PSRAM and add more log messages

* cleanup of init checks

* .

* .

* .

Co-authored-by: CaCO3 <caco@ruinelli.ch>

code/components/jomjol_helper/Helper.cpp

@@ -36,6 +36,8 @@ static const char* TAG = "HELPER";
 
 using namespace std;
 
+unsigned int systemStatus = 0;
+
 sdmmc_cid_t SDCardCid;
 sdmmc_csd_t SDCardCsd;
 
@@ -771,6 +773,38 @@ string getMac(void) {
     return macFormated;
 }
 
+
+void setSystemStatusFlag(SystemStatusFlag_t flag) {
+	systemStatus = systemStatus | flag; // set bit
+
+	char buf[20];
+	snprintf(buf, sizeof(buf), "0x%08X", getSystemStatus());
+    LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "New System Status: " + std::string(buf));
+}
+
+void clearSystemStatusFlag(SystemStatusFlag_t flag) {
+	systemStatus = systemStatus | ~flag; // clear bit
+
+	char buf[20];
+	snprintf(buf, sizeof(buf), "0x%08X", getSystemStatus());
+    LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "New System Status: " + std::string(buf));
+}
+
+int getSystemStatus(void) {
+    return systemStatus;
+}
+
+bool isSetSystemStatusFlag(SystemStatusFlag_t flag) {
+	//ESP_LOGE(TAG, "Flag (0x%08X) is set (0x%08X): %d", flag, systemStatus , ((systemStatus & flag) == flag));
+
+	if ((systemStatus & flag) == flag) {
+		return true;
+	}
+	else {
+		return false;
+	}
+}
+
 string getResetReason(void) {
 	std::string reasonText;
 

code/components/jomjol_helper/Helper.h

@@ -63,6 +63,27 @@ string getSDCardCapacity();
 string getSDCardSectorSize();
 
 string getMac(void);
+
+
+/* Error bit fields
+   One bit per error
+   Make sure it matches https://github.com/jomjol/AI-on-the-edge-device/wiki/Error-Codes */
+enum SystemStatusFlag_t {          // One bit per error
+    // First Byte
+    SYSTEM_STATUS_PSRAM_BAD         = 1 << 0, //  4, Critical Error
+    SYSTEM_STATUS_HEAP_TOO_SMALL    = 1 << 1, //  8, Critical Error
+    SYSTEM_STATUS_CAM_BAD           = 1 << 2, //  1, Critical Error
+
+    // Second Byte
+    SYSTEM_STATUS_CAM_FB_BAD        = 1 << (0+8), //  2, Flow still might work
+    SYSTEM_STATUS_NTP_BAD           = 1 << (1+8), // 16, Flow will work but time will be wrong
+};
+
+void setSystemStatusFlag(SystemStatusFlag_t flag);
+void clearSystemStatusFlag(SystemStatusFlag_t flag);
+int getSystemStatus(void);
+bool isSetSystemStatusFlag(SystemStatusFlag_t flag);
+
 string getResetReason(void);
 std::string getFormatedUptime(bool compact);
 

code/main/CMakeLists.txt

@@ -65,5 +65,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_SOURCE_DIR}/main/*.*)
 # idf_component_register(SRCS ${app_sources})
 
 idf_component_register(SRCS ${app_sources}
-                    INCLUDE_DIRS ".")
+                    INCLUDE_DIRS "."
-
+                   # REQUIRES esp_psram) # comming in IDF 5.0
+                    )

code/main/main.cpp

@@ -5,6 +5,9 @@
 
 #include "driver/gpio.h"
 #include "sdkconfig.h"
+//#include "esp_psram.h" // Comming in IDF 5.0, see https://docs.espressif.com/projects/esp-idf/en/v5.0-beta1/esp32/migration-guides/release-5.x/system.html?highlight=esp_psram_get_size
+#include "spiram.h"
+#include "esp_spiram.h"
 
 // SD-Card ////////////////////
 #include "nvs_flash.h"
@@ -122,7 +125,7 @@ bool Init_NVS_SDCard()
     return true;
 }
 
-void task_NoSDBlink(void *pvParameter)
+void task_MainInitError_blink(void *pvParameter)
 {
     gpio_pad_select_gpio(BLINK_GPIO);
     gpio_set_direction(BLINK_GPIO, GPIO_MODE_OUTPUT);  
@@ -147,7 +150,6 @@ void task_NoSDBlink(void *pvParameter)
 extern "C" void app_main(void)
 {
     TickType_t xDelay;
-    bool initSucessful = true;
 
     ESP_LOGI(TAG, "\n\n\n\n\n"); // Add mark on log to see when it restarted
     
@@ -160,9 +162,9 @@ extern "C" void app_main(void)
 
     if (!Init_NVS_SDCard())
     {
-        xTaskCreate(&task_NoSDBlink, "task_NoSDBlink", configMINIMAL_STACK_SIZE * 64, NULL, tskIDLE_PRIORITY+1, NULL);
+        xTaskCreate(&task_MainInitError_blink, "task_MainInitError_blink", configMINIMAL_STACK_SIZE * 64, NULL, tskIDLE_PRIORITY+1, NULL);
-        return;
+        return; // No way to continue without SD-Card!
-    };
+    }
 
     string versionFormated = "Branch: '" + std::string(GIT_BRANCH) + \
         "', Revision: " + std::string(GIT_REV) +", Date/Time: " + std::string(BUILD_TIME) + \
@@ -216,6 +218,7 @@ extern "C" void app_main(void)
 
     if (!setup_time()) {
         LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "NTP Initialization failed!");
+        setSystemStatusFlag(SYSTEM_STATUS_NTP_BAD);
     }
 
     setBootTime();
@@ -231,13 +234,32 @@ extern "C" void app_main(void)
     std::string zw = gettimestring("%Y%m%d-%H%M%S");
     ESP_LOGD(TAG, "time %s", zw.c_str());
 
+    /* Check if PSRAM can be initalized */
+    esp_err_t ret;
+    ret = esp_spiram_init();
+    if (ret == ESP_FAIL) { // Failed to init PSRAM, most likely not available or broken
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to initialize PSRAM (" + std::to_string(ret) + ")!");
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Either your device misses the PSRAM chip or it is broken!");
+        setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD);
+    }
+    else { // PSRAM init ok
+        /* Check if PSRAM provides at least 4 MB */
+        size_t psram_size = esp_spiram_get_size();        
+        // size_t psram_size = esp_psram_get_size(); // comming in IDF 5.0
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "The device has " + std::to_string(psram_size/1024/1024) + " MBytes of PSRAM");
+        if (psram_size < (4*1024*1024)) { // PSRAM is below 4 MBytes
+            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "At least 4 MBytes are required!");
+            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Does the device really have a 4 Mbytes PSRAM?");
+            setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD);
+        }
+    }
+
+    /* Check available Heap memory */
     size_t _hsize = getESPHeapSize();
-    if (_hsize < 4000000) // Check for a bit less than 4 MB (but clearly over 2 MB)
+    if (_hsize < 4000000) { // Check available Heap memory for a bit less than 4 MB (a test on a good device showed 4187558 bytes to be available)
-    {
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Not enough Heap memory available. Expected around 4 MBytes, but only " + std::to_string(_hsize) + " Bytes are available! That is not enough for this firmware!");
-        std::string _zws = "Not enough PSRAM available. Expected around 4 MBytes - available: " + std::to_string((float)_hsize/1024/1024) + " MBytes!";
+        setSystemStatusFlag(SYSTEM_STATUS_HEAP_TOO_SMALL);
-        _zws = _zws + "\nEither not initialized, too small (2 MByte only) or not present at all. Firmware cannot start!!";
+    } else { // Heap memory is ok
-        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, _zws);
-    } else { // Bad Camera Status, retry init   
         if (camStatus != ESP_OK) {
             LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Failed to initialize camera module, retrying...");
 
@@ -251,7 +273,7 @@ extern "C" void app_main(void)
             if (camStatus != ESP_OK) {
                 LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to initialize camera module!");
                 LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Check that your camera module is working and connected properly!");
-                initSucessful = false;
+                setSystemStatusFlag(SYSTEM_STATUS_CAM_BAD);
             }
         } else { // Test Camera            
             camera_fb_t * fb = esp_camera_fb_get();
@@ -259,8 +281,8 @@ extern "C" void app_main(void)
                 LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Camera Framebuffer cannot be initialized!");
                 /* Easiest would be to simply restart here and try again,
                    how ever there seem to be systems where it fails at startup but still work corectly later.
-                   Therefore we treat it still as successed!
+                   Therefore we treat it still as successed! */
-                   //initSucessful = false; */
+                   setSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD);
             }
             else {
                 esp_camera_fb_return(fb);   
@@ -269,8 +291,6 @@ extern "C" void app_main(void)
         }
     }
 
-
-
     xDelay = 2000 / portTICK_PERIOD_MS;
     ESP_LOGD(TAG, "main: sleep for: %ldms", (long) xDelay*10);
     vTaskDelay( xDelay ); 
@@ -291,17 +311,24 @@ extern "C" void app_main(void)
     ESP_LOGD(TAG, "vor reg server main");
     register_server_main_uri(server, "/sdcard");
 
-    if (initSucessful) {
+
+    /* Testing */
+    //setSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD);
+    //setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD);
+
+    /* Main Init has successed or only an error which allows to continue operation */
+    if (getSystemStatus() == 0) { // No error flag is set
         LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Initialization completed successfully!");
         ESP_LOGD(TAG, "vor do autostart");
         TFliteDoAutoStart();
     }
-    else { // Initialization failed
+    else if (isSetSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD) || // Non critical errors occured, we try to continue...
-        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart in 5 minutes!");
+        isSetSystemStatusFlag(SYSTEM_STATUS_NTP_BAD)) {
-        vTaskDelay(60*4000 / portTICK_RATE_MS); // Wait 4 minutes to give time to do an OTA or fetch the log
+        LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Initialization completed with errors, but trying to continue...");
-        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart in 1 minute!");
+        ESP_LOGD(TAG, "vor do autostart");
-        vTaskDelay(60*1000 / portTICK_RATE_MS); // Wait 1 minute to give time to do an OTA or fetch the log
+        TFliteDoAutoStart();
-        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart now!");
+    }
-        doReboot();
+    else { // Any other error is critical and makes running the flow impossible.
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Not starting flows!");
     }
 }

code/main/server_main.cpp

@@ -230,10 +230,30 @@ esp_err_t hello_main_handler(httpd_req_t *req)
         }
     }
 
-    if (filetosend == "/sdcard/html/index.html" && isSetupModusActive()) {
+    if (filetosend == "/sdcard/html/index.html") {
+        if (isSetSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD) || // Initialization failed with crritical errors!
+                isSetSystemStatusFlag(SYSTEM_STATUS_CAM_BAD)) {
+            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "We have a critical error, not serving main page!");
+
+            char buf[20];
+            std::string message = "<h1>AI on the Edge Device</h1><b>We have one or more critical errors:</b><br>";
+
+            for (int i = 0; i < 32; i++) {
+                if (isSetSystemStatusFlag((SystemStatusFlag_t)(1<<i))) {
+                    snprintf(buf, sizeof(buf), "0x%08X", 1<<i);
+                    message += std::string(buf) + "<br>";
+                }
+            }
+
+            message += "<br>Please check <a href=\"https://github.com/jomjol/AI-on-the-edge-device/wiki/Error-Codes\" target=_blank>github.com/jomjol/AI-on-the-edge-device/wiki/Error-Codes</a> for more information!";
+            httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, message.c_str());
+            return ESP_FAIL;
+        }
+        else if (isSetupModusActive()) {
             ESP_LOGD(TAG, "System is in setup mode --> index.html --> setup.html");
             filetosend = "/sdcard/html/setup.html";
         }
+    }
 
     ESP_LOGD(TAG, "Filename: %s", filename);
     

code/sdkconfig.defaults

@@ -36,6 +36,7 @@ CONFIG_SPIRAM_MEMTEST=y
 CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384
 CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=40960
 CONFIG_SPIRAM_CACHE_WORKAROUND=y
+CONFIG_SPIRAM_IGNORE_NOTFOUND=y
 
 CONFIG_ESP_INT_WDT_TIMEOUT_MS=300