Store preprocessed image with ROI to RAM (#1809)

* tflite model loading: error handling

* FlowAlignment: error handling

* CImageBasis+GetJPGStream : error handling

* store preprocessed ALG_ROI.jpg to memory

* Update

code/components/jomjol_controlcamera/ClassControllCamera.cpp

@@ -236,18 +236,18 @@ void CCamera::EnableAutoExposure(int flash_duration)
 
 esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
 {
-    string ftype;
-    int _size;
-
-    uint8_t *zwischenspeicher = NULL;
-
-
-    LEDOnOff(true);
-
 	#ifdef DEBUG_DETAIL_ON
 	    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Start");
 	#endif
 
+    _Image->EmptyImage(); //Delete previous stored raw image -> black image
+    
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG__SHOW_TAKE_IMAGE_PROCESS
+        tfliteflow.SetNewAlgROI(false);
+    #endif
+    
+    LEDOnOff(true);
+
     if (delay > 0) 
     {
         LightOnOff(true);
@@ -263,10 +263,10 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
     esp_camera_fb_return(fb);        
     fb = esp_camera_fb_get();
     if (!fb) {
-        ESP_LOGE(TAG, "CaptureToBasisImage: Capture Failed");
         LEDOnOff(false);
         LightOnOff(false);
 
+        ESP_LOGE(TAG, "CaptureToBasisImage: Capture Failed");
         LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "is not working anymore (CCamera::CaptureToBasisImage) - most probably caused by a hardware problem (instablility, ...). "
                 "System will reboot.");
         doReboot();
@@ -279,15 +279,8 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
         loadNextDemoImage(fb);
     }
 
-    _size = fb->len;
+    CImageBasis* _zwImage = new CImageBasis();
-    zwischenspeicher = (uint8_t*) malloc(_size);
+    _zwImage->LoadFromMemory(fb->buf, fb->len);
-    if (!zwischenspeicher)
-    {
-        ESP_LOGE(TAG, "Insufficient memory space for image in function CaptureToBasisImage()");
-        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Insufficient memory space for image in function CaptureToBasisImage()");
-    }
-    for (int i = 0; i < _size; ++i)
-        *(zwischenspeicher + i) = *(fb->buf + i);
     esp_camera_fb_return(fb);        
 
     #ifdef DEBUG_DETAIL_ON
@@ -302,12 +295,6 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
 //    TickType_t xDelay = 1000 / portTICK_PERIOD_MS;     
 //    vTaskDelay( xDelay );  // wait for power to recover
     
-    uint8_t * buf = NULL;
-
-    CImageBasis _zwImage;
-    _zwImage.LoadFromMemory(zwischenspeicher, _size);
-    free(zwischenspeicher);
-
     #ifdef DEBUG_DETAIL_ON
         LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LoadFromMemory");
     #endif
@@ -328,7 +315,7 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
         for (int y = 0; y < height; ++y)
         {
             p_target = _Image->rgb_image + (channels * (y * width + x));
-            p_source = _zwImage.rgb_image + (channels * (y * width + x));
+            p_source = _zwImage->rgb_image + (channels * (y * width + x));
             p_target[0] = p_source[0];
             p_target[1] = p_source[1];
             p_target[2] = p_source[2];
@@ -338,7 +325,7 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
         LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After Copy To Target");
     #endif
 
-    free(buf);
+    delete _zwImage;
 
     #ifdef DEBUG_DETAIL_ON
         LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Done");
@@ -604,7 +591,7 @@ void CCamera::GetCameraParameter(httpd_req_t *req, int &qual, framesize_t &resol
             if (qual < 0)
                 qual = 0;
         }
-    };
+    }
 }
 
 
@@ -674,6 +661,7 @@ bool CCamera::getCameraInitSuccessful()
     return CameraInitSuccessful;
 }
 
+
 std::vector<std::string> demoFiles;
 
 void CCamera::useDemoMode()

code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp

@@ -1,6 +1,7 @@
 #include "ClassFlowAlignment.h"
 #include "ClassFlowMakeImage.h"
 #include "ClassFlow.h"
+#include "server_tflite.h"
 
 #include "CRotateImage.h"
 #include "esp_log.h"
@@ -29,6 +30,9 @@ void ClassFlowAlignment::SetInitialParameter(void)
     AlignAndCutImage = NULL;
     ImageBasis = NULL;
     ImageTMP = NULL;
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG 
+    AlgROI = (ImageData*)heap_caps_malloc(sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);
+    #endif
     previousElement = NULL;
     disabled = false;
     SAD_criteria = 0.05;
@@ -161,6 +165,25 @@ string ClassFlowAlignment::getHTMLSingleStep(string host)
 
 bool ClassFlowAlignment::doFlow(string time) 
 {
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
+        if (!AlgROI)  // AlgROI needs to be allocated before ImageTMP to avoid heap fragmentation
+        {
+            AlgROI = (ImageData*)heap_caps_realloc(AlgROI, sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);     
+            if (!AlgROI) 
+            {
+                LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate AlgROI");
+                LogFile.WriteHeapInfo("ClassFlowAlignment-doFlow");
+                tfliteflow.SetNewAlgROI(false); // continue flow only with alg.jpg (no ROIs available)
+            }
+        }
+
+        if (AlgROI)
+        {
+            ImageBasis->writeToMemoryAsJPG((ImageData*)AlgROI, 90);
+            tfliteflow.SetNewAlgROI(true);
+        }
+    #endif
+
     if (!ImageTMP) 
     {
         ImageTMP = new CImageBasis(ImageBasis);
@@ -214,7 +237,14 @@ bool ClassFlowAlignment::doFlow(string time)
         SaveReferenceAlignmentValues();
     }
 
-    if (SaveAllFiles) AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/alg.jpg"));
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
+        if (AlgROI) {
+            DrawRef(ImageTMP);
+            tfliteflow.DigitalDrawROI(ImageTMP);
+            tfliteflow.AnalogDrawROI(ImageTMP);
+            ImageTMP->writeToMemoryAsJPG((ImageData*)AlgROI, 90);
+        }
+    #endif
     
     if (SaveAllFiles)
     {
@@ -224,7 +254,8 @@ bool ClassFlowAlignment::doFlow(string time)
             ImageTMP->width = ImageTMP->height;
             ImageTMP->height = _zw;
         }
-        DrawRef(ImageTMP);
+
+        AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/alg.jpg"));
         ImageTMP->SaveToFile(FormatFileName("/sdcard/img_tmp/alg_roi.jpg"));
     }
 

code/components/jomjol_flowcontroll/ClassFlowAlignment.h

@@ -34,6 +34,9 @@ protected:
 
 public:
     CImageBasis *ImageBasis, *ImageTMP;
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG 
+    ImageData *AlgROI;
+    #endif
     
     ClassFlowAlignment(std::vector<ClassFlow*>* lfc);
 

code/components/jomjol_flowcontroll/ClassFlowControll.cpp

@@ -141,6 +141,27 @@ t_CNNType ClassFlowControll::GetTypeAnalog()
 }
 
 
+#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
+void ClassFlowControll::DigitalDrawROI(CImageBasis *_zw)
+{
+    if (flowdigit)
+        flowdigit->DrawROI(_zw);
+}
+
+
+void ClassFlowControll::AnalogDrawROI(CImageBasis *_zw)
+{
+    if (flowanalog)
+        flowanalog->DrawROI(_zw);
+}
+
+
+void ClassFlowControll::SetNewAlgROI(bool _value)
+{
+    bNewAlgROI = _value;
+}
+#endif
+
 
 #ifdef ENABLE_MQTT
 string ClassFlowControll::GetMQTTMainTopic()
@@ -667,6 +688,27 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
         }
     }
     else if (_fn == "alg_roi.jpg") {
+        #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG      // no CImageBasis needed to create alg_roi.jpg (ca. 790kB less RAM)
+            if (bNewAlgROI) {
+                if (flowalignment && flowalignment->AlgROI) {
+                    httpd_resp_set_type(req, "image/jpeg");
+                    result = httpd_resp_send(req, (const char *)flowalignment->AlgROI->data, flowalignment->AlgROI->size);
+                }
+                else {
+                    LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "ClassFlowControll::GetJPGStream: alg_roi.jpg cannot be served");
+                    return ESP_FAIL;
+                }
+            }
+            else {
+                if (flowalignment && flowalignment->ImageBasis->ImageOkay()) {
+                    _send = flowalignment->ImageBasis;
+                }
+                else {
+                    httpd_resp_send(req, NULL, 0);
+                    return ESP_OK;
+                }
+            }
+        #else
             _send = new CImageBasis(flowalignment->ImageBasis);
 			
             if (_send->ImageOkay()) {
@@ -686,6 +728,7 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
                     return ESP_OK;
                 }
             }
+        #endif
     }
     else {
         std::vector<HTMLInfo*> htmlinfo;

code/components/jomjol_flowcontroll/ClassFlowControll.h

@@ -38,6 +38,9 @@ protected:
 	void SetInitialParameter(void);	
 	std::string aktstatus;
 	int aktRunNr;
+	#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG 
+	bool bNewAlgROI = false;
+	#endif
 
 public:
 	void InitFlow(std::string config);
@@ -51,12 +54,21 @@ public:
 	bool ReadParameter(FILE* pfile, string& aktparamgraph);	
 	string getJSON();
 	string getNumbersName();
+	#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG 
+	void SetNewAlgROI(bool _value);
+	#endif
 
 	string TranslateAktstatus(std::string _input);
+
 	#ifdef ENABLE_MQTT
 	string GetMQTTMainTopic();
 	#endif //ENABLE_MQTT
 
+	#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
+	void DigitalDrawROI(CImageBasis *_zw);
+	void AnalogDrawROI(CImageBasis *_zw);
+	#endif
+
 	esp_err_t GetJPGStream(std::string _fn, httpd_req_t *req);
 	esp_err_t SendRawJPG(httpd_req_t *req);
 
@@ -71,6 +83,7 @@ public:
 
 	t_CNNType GetTypeDigital();
 	t_CNNType GetTypeAnalog();
+	
 	#ifdef ENABLE_MQTT
 	bool StartMQTTService();
 	#endif //ENABLE_MQTT

code/components/jomjol_image_proc/CImageBasis.cpp

@@ -84,6 +84,19 @@ ImageData* CImageBasis::writeToMemoryAsJPG(const int quality)
 }
 
 
+void CImageBasis::writeToMemoryAsJPG(ImageData* i, const int quality)
+{
+    ImageData* ii = new ImageData;
+    
+    RGBImageLock();
+    stbi_write_jpg_to_func(writejpghelp, ii, width, height, channels, rgb_image, quality);
+    RGBImageRelease();
+
+    memCopy((uint8_t*) ii, (uint8_t*) i, sizeof(ImageData));
+    delete ii;
+}
+
+
 struct SendJPGHTTP
 {
     httpd_req_t *req;
@@ -380,6 +393,28 @@ void CImageBasis::CreateEmptyImage(int _width, int _height, int _channels)
 }
 
 
+void CImageBasis::EmptyImage()
+{
+    #ifdef DEBUG_DETAIL_ON 
+        LogFile.WriteHeapInfo("CImageBasis::EmptyImage");
+    #endif
+
+    stbi_uc* p_source;
+
+    RGBImageLock();
+
+    for (int x = 0; x < width; ++x)
+        for (int y = 0; y < height; ++y)
+        {
+            p_source = rgb_image + (channels * (y * width + x));
+            for (int _channels = 0; _channels < channels; ++_channels)
+                p_source[_channels] = (uint8_t) 0;
+        }
+
+    RGBImageRelease();
+}
+
+
 void CImageBasis::LoadFromMemory(stbi_uc *_buffer, int len)
 {
     RGBImageLock();

code/components/jomjol_image_proc/CImageBasis.h

@@ -61,6 +61,7 @@ class CImageBasis
         void SetIndepended(){externalImage = false;};
 
         void CreateEmptyImage(int _width, int _height, int _channels);
+        void EmptyImage();
 
 
         CImageBasis();
@@ -75,6 +76,7 @@ class CImageBasis
         void LoadFromMemory(stbi_uc *_buffer, int len);
 
         ImageData* writeToMemoryAsJPG(const int quality = 90);
+        void writeToMemoryAsJPG(ImageData* ii, const int quality = 90);
 
         esp_err_t SendJPGtoHTTP(httpd_req_t *req, const int quality = 90);   
 

code/include/defines.h

@@ -88,6 +88,10 @@
     #define READOUT_TYPE_RAWVALUE 2
     #define READOUT_TYPE_ERROR 3
 
+    //ClassFlowControll: Serve alg_roi.jpg from memory as JPG
+    #define ALGROI_LOAD_FROM_MEM_AS_JPG // Load ALG_ROI.JPG as rendered JPG from RAM
+    #define ALGROI_LOAD_FROM_MEM_AS_JPG__SHOW_TAKE_IMAGE_PROCESS // Show take image image processing on webinterface (overview.html)
+
     //ClassFlowMQTT
     #define LWT_TOPIC        "connection"
     #define LWT_CONNECTED    "connected"