From 24f10b468362cd4601a71128a3c868cf44492ccc Mon Sep 17 00:00:00 2001 From: CaCO3 Date: Mon, 12 Dec 2022 16:57:14 +0100 Subject: [PATCH] Continue boot on bad PSRAM (#1558) * add sensor status, show special index/setup page in case of a critical error * continue booting on PSRAM error * Update main.cpp added log messages * init PSRAM and add more log messages * cleanup of init checks * . * . * . Co-authored-by: CaCO3 --- code/components/jomjol_helper/Helper.cpp | 34 +++++++++++ code/components/jomjol_helper/Helper.h | 21 +++++++ code/main/CMakeLists.txt | 5 +- code/main/main.cpp | 75 ++++++++++++++++-------- code/main/server_main.cpp | 26 +++++++- code/sdkconfig.defaults | 1 + 6 files changed, 133 insertions(+), 29 deletions(-) diff --git a/code/components/jomjol_helper/Helper.cpp b/code/components/jomjol_helper/Helper.cpp index c35642ed..1e7e28a1 100644 --- a/code/components/jomjol_helper/Helper.cpp +++ b/code/components/jomjol_helper/Helper.cpp @@ -36,6 +36,8 @@ static const char* TAG = "HELPER"; using namespace std; +unsigned int systemStatus = 0; + sdmmc_cid_t SDCardCid; sdmmc_csd_t SDCardCsd; @@ -771,6 +773,38 @@ string getMac(void) { return macFormated; } + +void setSystemStatusFlag(SystemStatusFlag_t flag) { + systemStatus = systemStatus | flag; // set bit + + char buf[20]; + snprintf(buf, sizeof(buf), "0x%08X", getSystemStatus()); + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "New System Status: " + std::string(buf)); +} + +void clearSystemStatusFlag(SystemStatusFlag_t flag) { + systemStatus = systemStatus | ~flag; // clear bit + + char buf[20]; + snprintf(buf, sizeof(buf), "0x%08X", getSystemStatus()); + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "New System Status: " + std::string(buf)); +} + +int getSystemStatus(void) { + return systemStatus; +} + +bool isSetSystemStatusFlag(SystemStatusFlag_t flag) { + //ESP_LOGE(TAG, "Flag (0x%08X) is set (0x%08X): %d", flag, systemStatus , ((systemStatus & flag) == flag)); + + if ((systemStatus & flag) == flag) { + return true; + } + else { + return false; + } +} + string getResetReason(void) { std::string reasonText; diff --git a/code/components/jomjol_helper/Helper.h b/code/components/jomjol_helper/Helper.h index 2f2cb05f..aba14b83 100644 --- a/code/components/jomjol_helper/Helper.h +++ b/code/components/jomjol_helper/Helper.h @@ -63,6 +63,27 @@ string getSDCardCapacity(); string getSDCardSectorSize(); string getMac(void); + + +/* Error bit fields + One bit per error + Make sure it matches https://github.com/jomjol/AI-on-the-edge-device/wiki/Error-Codes */ +enum SystemStatusFlag_t { // One bit per error + // First Byte + SYSTEM_STATUS_PSRAM_BAD = 1 << 0, // 4, Critical Error + SYSTEM_STATUS_HEAP_TOO_SMALL = 1 << 1, // 8, Critical Error + SYSTEM_STATUS_CAM_BAD = 1 << 2, // 1, Critical Error + + // Second Byte + SYSTEM_STATUS_CAM_FB_BAD = 1 << (0+8), // 2, Flow still might work + SYSTEM_STATUS_NTP_BAD = 1 << (1+8), // 16, Flow will work but time will be wrong +}; + +void setSystemStatusFlag(SystemStatusFlag_t flag); +void clearSystemStatusFlag(SystemStatusFlag_t flag); +int getSystemStatus(void); +bool isSetSystemStatusFlag(SystemStatusFlag_t flag); + string getResetReason(void); std::string getFormatedUptime(bool compact); diff --git a/code/main/CMakeLists.txt b/code/main/CMakeLists.txt index 4776f371..e2572d5a 100644 --- a/code/main/CMakeLists.txt +++ b/code/main/CMakeLists.txt @@ -65,5 +65,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_SOURCE_DIR}/main/*.*) # idf_component_register(SRCS ${app_sources}) idf_component_register(SRCS ${app_sources} - INCLUDE_DIRS ".") - + INCLUDE_DIRS "." + # REQUIRES esp_psram) # comming in IDF 5.0 + ) \ No newline at end of file diff --git a/code/main/main.cpp b/code/main/main.cpp index 614967e2..700d3605 100644 --- a/code/main/main.cpp +++ b/code/main/main.cpp @@ -5,6 +5,9 @@ #include "driver/gpio.h" #include "sdkconfig.h" +//#include "esp_psram.h" // Comming in IDF 5.0, see https://docs.espressif.com/projects/esp-idf/en/v5.0-beta1/esp32/migration-guides/release-5.x/system.html?highlight=esp_psram_get_size +#include "spiram.h" +#include "esp_spiram.h" // SD-Card //////////////////// #include "nvs_flash.h" @@ -122,7 +125,7 @@ bool Init_NVS_SDCard() return true; } -void task_NoSDBlink(void *pvParameter) +void task_MainInitError_blink(void *pvParameter) { gpio_pad_select_gpio(BLINK_GPIO); gpio_set_direction(BLINK_GPIO, GPIO_MODE_OUTPUT); @@ -147,7 +150,6 @@ void task_NoSDBlink(void *pvParameter) extern "C" void app_main(void) { TickType_t xDelay; - bool initSucessful = true; ESP_LOGI(TAG, "\n\n\n\n\n"); // Add mark on log to see when it restarted @@ -160,9 +162,9 @@ extern "C" void app_main(void) if (!Init_NVS_SDCard()) { - xTaskCreate(&task_NoSDBlink, "task_NoSDBlink", configMINIMAL_STACK_SIZE * 64, NULL, tskIDLE_PRIORITY+1, NULL); - return; - }; + xTaskCreate(&task_MainInitError_blink, "task_MainInitError_blink", configMINIMAL_STACK_SIZE * 64, NULL, tskIDLE_PRIORITY+1, NULL); + return; // No way to continue without SD-Card! + } string versionFormated = "Branch: '" + std::string(GIT_BRANCH) + \ "', Revision: " + std::string(GIT_REV) +", Date/Time: " + std::string(BUILD_TIME) + \ @@ -216,6 +218,7 @@ extern "C" void app_main(void) if (!setup_time()) { LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "NTP Initialization failed!"); + setSystemStatusFlag(SYSTEM_STATUS_NTP_BAD); } setBootTime(); @@ -231,13 +234,32 @@ extern "C" void app_main(void) std::string zw = gettimestring("%Y%m%d-%H%M%S"); ESP_LOGD(TAG, "time %s", zw.c_str()); + /* Check if PSRAM can be initalized */ + esp_err_t ret; + ret = esp_spiram_init(); + if (ret == ESP_FAIL) { // Failed to init PSRAM, most likely not available or broken + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to initialize PSRAM (" + std::to_string(ret) + ")!"); + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Either your device misses the PSRAM chip or it is broken!"); + setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD); + } + else { // PSRAM init ok + /* Check if PSRAM provides at least 4 MB */ + size_t psram_size = esp_spiram_get_size(); + // size_t psram_size = esp_psram_get_size(); // comming in IDF 5.0 + LogFile.WriteToFile(ESP_LOG_INFO, TAG, "The device has " + std::to_string(psram_size/1024/1024) + " MBytes of PSRAM"); + if (psram_size < (4*1024*1024)) { // PSRAM is below 4 MBytes + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "At least 4 MBytes are required!"); + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Does the device really have a 4 Mbytes PSRAM?"); + setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD); + } + } + + /* Check available Heap memory */ size_t _hsize = getESPHeapSize(); - if (_hsize < 4000000) // Check for a bit less than 4 MB (but clearly over 2 MB) - { - std::string _zws = "Not enough PSRAM available. Expected around 4 MBytes - available: " + std::to_string((float)_hsize/1024/1024) + " MBytes!"; - _zws = _zws + "\nEither not initialized, too small (2 MByte only) or not present at all. Firmware cannot start!!"; - LogFile.WriteToFile(ESP_LOG_ERROR, TAG, _zws); - } else { // Bad Camera Status, retry init + if (_hsize < 4000000) { // Check available Heap memory for a bit less than 4 MB (a test on a good device showed 4187558 bytes to be available) + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Not enough Heap memory available. Expected around 4 MBytes, but only " + std::to_string(_hsize) + " Bytes are available! That is not enough for this firmware!"); + setSystemStatusFlag(SYSTEM_STATUS_HEAP_TOO_SMALL); + } else { // Heap memory is ok if (camStatus != ESP_OK) { LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Failed to initialize camera module, retrying..."); @@ -251,7 +273,7 @@ extern "C" void app_main(void) if (camStatus != ESP_OK) { LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to initialize camera module!"); LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Check that your camera module is working and connected properly!"); - initSucessful = false; + setSystemStatusFlag(SYSTEM_STATUS_CAM_BAD); } } else { // Test Camera camera_fb_t * fb = esp_camera_fb_get(); @@ -259,8 +281,8 @@ extern "C" void app_main(void) LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Camera Framebuffer cannot be initialized!"); /* Easiest would be to simply restart here and try again, how ever there seem to be systems where it fails at startup but still work corectly later. - Therefore we treat it still as successed! - //initSucessful = false; */ + Therefore we treat it still as successed! */ + setSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD); } else { esp_camera_fb_return(fb); @@ -269,8 +291,6 @@ extern "C" void app_main(void) } } - - xDelay = 2000 / portTICK_PERIOD_MS; ESP_LOGD(TAG, "main: sleep for: %ldms", (long) xDelay*10); vTaskDelay( xDelay ); @@ -291,17 +311,24 @@ extern "C" void app_main(void) ESP_LOGD(TAG, "vor reg server main"); register_server_main_uri(server, "/sdcard"); - if (initSucessful) { + + /* Testing */ + //setSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD); + //setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD); + + /* Main Init has successed or only an error which allows to continue operation */ + if (getSystemStatus() == 0) { // No error flag is set LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Initialization completed successfully!"); ESP_LOGD(TAG, "vor do autostart"); TFliteDoAutoStart(); } - else { // Initialization failed - LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart in 5 minutes!"); - vTaskDelay(60*4000 / portTICK_RATE_MS); // Wait 4 minutes to give time to do an OTA or fetch the log - LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart in 1 minute!"); - vTaskDelay(60*1000 / portTICK_RATE_MS); // Wait 1 minute to give time to do an OTA or fetch the log - LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart now!"); - doReboot(); + else if (isSetSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD) || // Non critical errors occured, we try to continue... + isSetSystemStatusFlag(SYSTEM_STATUS_NTP_BAD)) { + LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Initialization completed with errors, but trying to continue..."); + ESP_LOGD(TAG, "vor do autostart"); + TFliteDoAutoStart(); + } + else { // Any other error is critical and makes running the flow impossible. + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Not starting flows!"); } } diff --git a/code/main/server_main.cpp b/code/main/server_main.cpp index f713114d..4c2654bb 100644 --- a/code/main/server_main.cpp +++ b/code/main/server_main.cpp @@ -230,9 +230,29 @@ esp_err_t hello_main_handler(httpd_req_t *req) } } - if (filetosend == "/sdcard/html/index.html" && isSetupModusActive()) { - ESP_LOGD(TAG, "System is in setup mode --> index.html --> setup.html"); - filetosend = "/sdcard/html/setup.html"; + if (filetosend == "/sdcard/html/index.html") { + if (isSetSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD) || // Initialization failed with crritical errors! + isSetSystemStatusFlag(SYSTEM_STATUS_CAM_BAD)) { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "We have a critical error, not serving main page!"); + + char buf[20]; + std::string message = "

AI on the Edge Device

We have one or more critical errors:
"; + + for (int i = 0; i < 32; i++) { + if (isSetSystemStatusFlag((SystemStatusFlag_t)(1<"; + } + } + + message += "
Please check github.com/jomjol/AI-on-the-edge-device/wiki/Error-Codes for more information!"; + httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, message.c_str()); + return ESP_FAIL; + } + else if (isSetupModusActive()) { + ESP_LOGD(TAG, "System is in setup mode --> index.html --> setup.html"); + filetosend = "/sdcard/html/setup.html"; + } } ESP_LOGD(TAG, "Filename: %s", filename); diff --git a/code/sdkconfig.defaults b/code/sdkconfig.defaults index 34864d80..95caac5f 100644 --- a/code/sdkconfig.defaults +++ b/code/sdkconfig.defaults @@ -36,6 +36,7 @@ CONFIG_SPIRAM_MEMTEST=y CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384 CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=40960 CONFIG_SPIRAM_CACHE_WORKAROUND=y +CONFIG_SPIRAM_IGNORE_NOTFOUND=y CONFIG_ESP_INT_WDT_TIMEOUT_MS=300