Shared PSRAM memory (#2285)

* enable PSRAM logging

* add extra functions for psram shared memroy handling

* CImageBasis objects still should used dynamic memory (eg. rawImage), haw ever tmpImage must be placed inside the shared memory

* Place all STBI allocs inside the shared memory

* The models are placed in the shared PSRAM reagion and must be allocated through the dedicated functions

* .

* renaming

* fix cast warning

* add flag to switch STBI PSRAM usage

* improve PSRAM shared handling

* reserve shared PSRAM as early as possible

* init logging eralier so we can use it in PSRAM shared alloc

* move Wifi_LWIP, BSS_SEG and MQTT Outbox into PSRAM to ffree internal memory

* Check if model fits into reserved shared memory

* Update code/components/jomjol_tfliteclass/CTfLiteClass.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_flowcontroll/ClassFlowControll.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_image_proc/CImageBasis.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* Update code/components/jomjol_helper/psram.cpp

* .

* .

* .

* .

* Korrektur Merge Conflict in main.cpp

---------

Co-authored-by: CaCO3 <caco@ruinelli.ch>
Co-authored-by: jomjol <30766535+jomjol@users.noreply.github.com>

code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp

@@ -190,10 +190,10 @@ bool ClassFlowAlignment::doFlow(string time)
 
     if (!ImageTMP) 
     {
-        ImageTMP = new CImageBasis("ImageTMP", ImageBasis);
+        ImageTMP = new CImageBasis("tmpImage", ImageBasis); // Make sure the name does not get change, it is relevant for the PSRAM allocation!
         if (!ImageTMP) 
         {
-            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate ImageTMP -> Exec this round aborted!");
+            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate tmpImage -> Exec this round aborted!");
             LogFile.WriteHeapInfo("ClassFlowAlignment-doFlow");
             return false;
         }

code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp

@@ -374,8 +374,11 @@ bool ClassFlowCNNGeneral::ReadParameter(FILE* pfile, string& aktparamgraph)
         }
     }
 
-    if (!getNetworkParameter())
+    if (!getNetworkParameter()) {
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "An error occured on setting up the Network -> Disabling it!");
+        disabled = true; // An error occured, disable this CNN!
         return false;
+    }
 
 
     for (int _ana = 0; _ana < GENERAL.size(); ++_ana)

code/components/jomjol_flowcontroll/ClassFlowControll.cpp

@@ -373,7 +373,7 @@ bool ClassFlowControll::doFlow(string time)
         zw_time = getCurrentTimeString("%H:%M:%S");
         aktstatus = TranslateAktstatus(FlowControll[i]->name());
         aktstatusWithTime = aktstatus + " (" + zw_time + ")";
-        //LogFile.WriteToFile(ESP_LOG_INFO, TAG, aktstatusWithTime);
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Status: " + aktstatusWithTime);
         #ifdef ENABLE_MQTT
             MQTTPublish(mqttServer_getMainTopic() + "/" + "status", aktstatus, qos, false);
         #endif //ENABLE_MQTT

code/components/jomjol_flowcontroll/ClassFlowTakeImage.cpp

@@ -8,6 +8,7 @@
 #include "esp_wifi.h"
 #include "esp_log.h"
 #include "../../include/defines.h"
+#include "psram.h"
 
 #include <time.h>
 
@@ -183,6 +184,8 @@ string ClassFlowTakeImage::getHTMLSingleStep(string host)
 
 bool ClassFlowTakeImage::doFlow(string zwtime)
 {
+    psram_init_shared_memory_for_take_image_step();
+
     string logPath = CreateLogFolder(zwtime);
 
     int flash_duration = (int) (waitbeforepicture * 1000);

code/components/jomjol_helper/psram.cpp

@@ -1,11 +1,128 @@
 #include "ClassLogFile.h"
-#include "esp_heap_caps.h"
+#include "../../include/defines.h"
+#include "psram.h"
 
 static const char* TAG = "PSRAM";
 
 using namespace std;
 
 
+void *shared_region = NULL;
+uint32_t allocatedBytesForSTBI = 0;
+
+
+/** Reserve a large block in the PSRAM which will be shared between the different steps.
+ * Each step uses it differently but only wiuthin itself. */
+bool reserve_psram_shared_region(void) {
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Allocating shared PSRAM region (" + 
+            std::to_string(TENSOR_ARENA_SIZE + MAX_MODEL_SIZE) + " bytes)...");
+    shared_region = malloc_psram_heap("Shared PSRAM region", TENSOR_ARENA_SIZE + MAX_MODEL_SIZE, 
+            MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+
+    if (shared_region == NULL) {
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to allocating shared PSRAM region!");
+        return false;
+    }
+    else {
+        return true;
+    }
+}
+
+
+
+/*******************************************************************
+ * Memory used in Take Image (STBI)
+ *******************************************************************/
+void psram_init_shared_memory_for_take_image_step(void) {
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Init shared memory for step 'Take Image' (STBI buffers)");
+    allocatedBytesForSTBI = 0;
+}
+
+
+void *psram_reserve_shared_stbi_memory(size_t size) {
+    /* Only large buffers should be placed in the shared PSRAM 
+     * If we also place all smaller STBI buffers here, we get artefacts for some reasons. */
+    if (size >= 100000) {
+        if ((allocatedBytesForSTBI + size) > TENSOR_ARENA_SIZE + MAX_MODEL_SIZE) { // Check if it still fits in the shared region
+            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Shared memory in PSRAM too small (STBI) to fit additional " + 
+                    std::to_string(size) + "bytes! Available: " + std::to_string(TENSOR_ARENA_SIZE + MAX_MODEL_SIZE - allocatedBytesForSTBI) + " bytes!");
+            return NULL;
+        }
+        
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Allocating memory (" + std::to_string(size) + " bytes) for STBI (use shared memory in PSRAM)...");
+        allocatedBytesForSTBI += size;
+        return (uint8_t *)shared_region + allocatedBytesForSTBI - size;
+    }
+    else { // Normal PSRAM
+        return malloc_psram_heap("STBI", size, MALLOC_CAP_SPIRAM);
+    }
+}
+
+
+void *psram_reallocate_shared_stbi_memory(void *ptr, size_t newsize) {
+    char buf[20];
+    sprintf(buf, "%p", ptr);
+    LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "STBI requested realloc for " + std::string(buf) + " but this is currently unsupported!");
+    return NULL;
+}
+
+
+void psram_free_shared_stbi_memory(void *p) {
+    if ((p >= shared_region) && (p <= ((uint8_t *)shared_region + allocatedBytesForSTBI))) { // was allocated inside the shared memory
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Part of shared memory used for STBI (PSRAM, part of shared memory) is free again");
+    }
+    else { // Normal PSRAM
+        free_psram_heap("STBI", p);
+    }
+}
+
+
+
+/*******************************************************************
+ * Memory used in Aligning Step 
+ * During this step we only use the shared part of the PSRAM
+ * for the tmpImage. 
+ *******************************************************************/
+void *psram_reserve_shared_tmp_image_memory(void) {  
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Allocating tmpImage (" + std::to_string(IMAGE_SIZE) + " bytes, use shared memory in PSRAM)...");
+    return shared_region; // Use 1th part of the shared memory for the tmpImage (only user)
+}
+
+
+void psram_free_shared_temp_image_memory(void) {
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Shared memory used for tmpImage (PSRAM, part of shared memory) is free again");
+}
+
+
+
+/*******************************************************************
+ * Memory used in Digitalization Steps
+ * During this step we only use the shared part of the PSRAM for the
+ * Tensor Arena and one of the Models.
+ * The shared memory is large enough for the largest model and the
+ * Tensor Arena. Therefore we do not need to monitor the usage.
+ *******************************************************************/
+void *psram_get_shared_tensor_arena_memory(void) {
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Allocating Tensor Arena (" + std::to_string(TENSOR_ARENA_SIZE) + " bytes, use shared memory in PSRAM)...");
+    return shared_region; // Use 1th part of the shared memory for Tensor
+}
+
+
+void *psram_get_shared_model_memory(void) {
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Allocating Model memory (" + std::to_string(MAX_MODEL_SIZE) + " bytes, use shared memory in PSRAM)...");
+    return (uint8_t *)shared_region + TENSOR_ARENA_SIZE; // Use 2nd part of the shared memory (after Tensor Arena) for the model
+}
+
+
+void psram_free_shared_tensor_arena_and_model_memory(void) {
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Shared memory used for Tensor Arena and model (PSRAM, part of shared memory) is free again");
+}
+
+
+
+/*******************************************************************
+ * General
+ *******************************************************************/
 void *malloc_psram_heap(std::string name, size_t size, uint32_t caps) {
 	void *ptr;
 
@@ -21,12 +138,25 @@ void *malloc_psram_heap(std::string name, size_t size, uint32_t caps) {
 }
 
 
+void *realloc_psram_heap(std::string name, void *ptr, size_t size, uint32_t caps) {
+	ptr = heap_caps_realloc(ptr, size, caps);
+    if (ptr != NULL) {
+	    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Reallocated " + to_string(size) + " bytes in PSRAM for '" + name + "'");
+	}
+    else {
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to reallocate " + to_string(size) + " bytes in PSRAM for '" + name + "'!");
+    }
+
+	return ptr;
+}
+
+
 void *calloc_psram_heap(std::string name, size_t n, size_t size, uint32_t caps) {
 	void *ptr;
 
 	ptr = heap_caps_calloc(n, size, caps);
     if (ptr != NULL) {
-	    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Allocated " + to_string(size) + " bytes in PSRAM for '" + name + "'");
+	    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Allocated " + to_string(size) + " bytes in PSRAM for '" + name + "'");
 	}
     else {
         LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to allocate " + to_string(size) + " bytes in PSRAM for '" + name + "'!");
@@ -39,4 +169,4 @@ void *calloc_psram_heap(std::string name, size_t n, size_t size, uint32_t caps)
 void free_psram_heap(std::string name, void *ptr) {
     LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Freeing memory in PSRAM used for '" + name + "'...");
     heap_caps_free(ptr);
-}
+}

code/components/jomjol_helper/psram.h

@@ -1,7 +1,34 @@
+#pragma once
+#ifndef PSRAM_h
+#define PSRAM_h
 
 #include "esp_heap_caps.h"
 
+
+bool reserve_psram_shared_region(void);
+
+
+/* Memory used in Take Image Step */
+void psram_init_shared_memory_for_take_image_step(void);
+void *psram_reserve_shared_stbi_memory(size_t size);
+void *psram_reallocate_shared_stbi_memory(void *ptr, size_t newsize);
+void psram_free_shared_stbi_memory(void *p);
+
+
+/* Memory used in Aligning Step */
+void *psram_reserve_shared_tmp_image_memory(void);
+void psram_free_shared_temp_image_memory(void);
+
+/* Memory used in Digitalization Steps */
+void *psram_get_shared_tensor_arena_memory(void);
+void *psram_get_shared_model_memory(void);
+void psram_free_shared_tensor_arena_and_model_memory(void);
+
+/* General */
 void *malloc_psram_heap(std::string name, size_t size, uint32_t caps);
+void *realloc_psram_heap(std::string name, void *ptr, size_t size, uint32_t caps);
 void *calloc_psram_heap(std::string name, size_t n, size_t size, uint32_t caps);
 
-void free_psram_heap(std::string name, void *ptr);
+void free_psram_heap(std::string name, void *ptr);
+
+#endif // PSRAM_h

code/components/jomjol_image_proc/CImageBasis.cpp

@@ -477,7 +477,13 @@ CImageBasis::CImageBasis(string _name, CImageBasis *_copyfrom)
 
     memsize = width * height * channels;
 
-    rgb_image = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->CImageBasis (" + name + ")", memsize, MALLOC_CAP_SPIRAM);
+
+    if (name == "tmpImage") {
+        rgb_image = (unsigned char*)psram_reserve_shared_tmp_image_memory();
+    }
+    else {
+        rgb_image = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->CImageBasis (" + name + ")", memsize, MALLOC_CAP_SPIRAM);
+    }
 
     if (rgb_image == NULL)
     {
@@ -618,9 +624,20 @@ CImageBasis::~CImageBasis()
 {
     RGBImageLock();
 
+
     if (!externalImage) {
-        //stbi_image_free(rgb_image);
-        free_psram_heap(std::string(TAG) + "->CImageBasis (" + name + ", " + to_string(memsize) + ")", rgb_image);
+        if (name == "tmpImage") { // This image should be placed in the shared part of PSRAM
+            psram_free_shared_temp_image_memory();
+        }
+        else { // All other images are much smaller and can go into the normal PSRAM region
+            //stbi_image_free(rgb_image);
+            if (memsize == 0) {
+                LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Not freeing (" + name + " as there was never PSRAM allocated for it)");
+            }
+            else {
+                free_psram_heap(std::string(TAG) + "->CImageBasis (" + name + ", " + to_string(memsize) + ")", rgb_image);
+            }
+        }
     }
 
     RGBImageRelease();

code/components/jomjol_image_proc/make_stb.cpp

@@ -1,8 +1,23 @@
 #include <stdint.h>
 #include <string>
+#include "psram.h"
 
 #include "../../include/defines.h"
 
+
+#define USE_SHARED_PSRAM_FOR_STBI
+
+#ifdef USE_SHARED_PSRAM_FOR_STBI
+#define STBI_MALLOC(sz)           psram_reserve_shared_stbi_memory(sz)
+#define STBI_REALLOC(p,newsz)     psram_reallocate_shared_stbi_memory(p, newsz)
+#define STBI_FREE(p)              psram_free_shared_stbi_memory(p)
+#else // Use normal PSRAM
+#define STBI_MALLOC(sz)           malloc_psram_heap("STBI", sz, MALLOC_CAP_SPIRAM)
+#define STBI_REALLOC(p,newsz)     realloc_psram_heap("STBI", p, newsz, MALLOC_CAP_SPIRAM)
+#define STBI_FREE(p)              free_psram_heap("STBI", p)
+#endif
+
+
 #define STB_IMAGE_IMPLEMENTATION
 #include "../stb/stb_image.h"
 

code/components/jomjol_tfliteclass/CTfLiteClass.cpp

@@ -243,15 +243,22 @@ bool CTfLiteClass::ReadFileToModel(std::string _fn)
 
     if (size == -1)
     {
-        ESP_LOGE(TAG, "CTfLiteClass::ReadFileToModel: Model file doesn't exist: %s", _fn.c_str());
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Model file doesn't exist: " + _fn + "!");
         return false;
     }
+    else if(size > MAX_MODEL_SIZE) {
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Unable to load model '" + _fn + "'! It does not fit in the reserved shared memory in PSRAM!");
+        return false;
+    }
+
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Loading Model " + _fn + " /size: " + std::to_string(size) + " bytes...");
+
 
 #ifdef DEBUG_DETAIL_ON      
         LogFile.WriteHeapInfo("CTLiteClass::Alloc modelfile start");
 #endif
 
-    modelfile = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->modelfile", size, MALLOC_CAP_SPIRAM);
+    modelfile = (unsigned char*)psram_get_shared_model_memory();
   
 	  if(modelfile != NULL) 
     {
@@ -304,9 +311,9 @@ CTfLiteClass::CTfLiteClass()
     this->modelfile = NULL;
     this->interpreter = nullptr;
     this->input = nullptr;
-    this->output = nullptr;  
-    this->kTensorArenaSize = 800 * 1024;   /// according to testfile: 108000 - so far 600;; 2021-09-11: 200 * 1024
-    this->tensor_arena = (uint8_t*)malloc_psram_heap(std::string(TAG) + "->tensor_arena", kTensorArenaSize, MALLOC_CAP_SPIRAM);
+    this->output = nullptr;
+    this->kTensorArenaSize = TENSOR_ARENA_SIZE;
+    this->tensor_arena = (uint8_t*)psram_get_shared_tensor_arena_memory();
 }
 
 
@@ -315,8 +322,7 @@ CTfLiteClass::~CTfLiteClass()
   delete this->interpreter;
   delete this->error_reporter;
 
-  free_psram_heap(std::string(TAG) + "->modelfile", modelfile);
-  free_psram_heap(std::string(TAG) + "->tensor_arena", this->tensor_arena);
+  psram_free_shared_tensor_arena_and_model_memory();
 }