diff --git a/code/components/jomjol_controlcamera/ClassControllCamera.cpp b/code/components/jomjol_controlcamera/ClassControllCamera.cpp index b849c500..6a3b84bd 100644 --- a/code/components/jomjol_controlcamera/ClassControllCamera.cpp +++ b/code/components/jomjol_controlcamera/ClassControllCamera.cpp @@ -236,18 +236,18 @@ void CCamera::EnableAutoExposure(int flash_duration) esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay) { - string ftype; - int _size; - - uint8_t *zwischenspeicher = NULL; - + #ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Start"); + #endif + _Image->EmptyImage(); //Delete previous stored raw image -> black image + + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG__SHOW_TAKE_IMAGE_PROCESS + tfliteflow.SetNewAlgROI(false); + #endif + LEDOnOff(true); -#ifdef DEBUG_DETAIL_ON - LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Start"); -#endif - if (delay > 0) { LightOnOff(true); @@ -255,18 +255,18 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay) vTaskDelay( xDelay ); } -#ifdef DEBUG_DETAIL_ON - LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LightOn"); -#endif + #ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LightOn"); + #endif camera_fb_t * fb = esp_camera_fb_get(); esp_camera_fb_return(fb); fb = esp_camera_fb_get(); if (!fb) { - ESP_LOGE(TAG, "CaptureToBasisImage: Capture Failed"); LEDOnOff(false); LightOnOff(false); + ESP_LOGE(TAG, "CaptureToBasisImage: Capture Failed"); LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "is not working anymore (CCamera::CaptureToBasisImage) - most probably caused by a hardware problem (instablility, ...). " "System will reboot."); doReboot(); @@ -279,20 +279,13 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay) loadNextDemoImage(fb); } - _size = fb->len; - zwischenspeicher = (uint8_t*) malloc(_size); - if (!zwischenspeicher) - { - ESP_LOGE(TAG, "Insufficient memory space for image in function CaptureToBasisImage()"); - LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Insufficient memory space for image in function CaptureToBasisImage()"); - } - for (int i = 0; i < _size; ++i) - *(zwischenspeicher + i) = *(fb->buf + i); + CImageBasis* _zwImage = new CImageBasis(); + _zwImage->LoadFromMemory(fb->buf, fb->len); esp_camera_fb_return(fb); -#ifdef DEBUG_DETAIL_ON - LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After fb_get"); -#endif + #ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After fb_get"); + #endif LEDOnOff(false); @@ -302,15 +295,9 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay) // TickType_t xDelay = 1000 / portTICK_PERIOD_MS; // vTaskDelay( xDelay ); // wait for power to recover - uint8_t * buf = NULL; - - CImageBasis _zwImage; - _zwImage.LoadFromMemory(zwischenspeicher, _size); - free(zwischenspeicher); - -#ifdef DEBUG_DETAIL_ON - LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LoadFromMemory"); -#endif + #ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LoadFromMemory"); + #endif stbi_uc* p_target; stbi_uc* p_source; @@ -318,31 +305,31 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay) int width = image_width; int height = image_height; -#ifdef DEBUG_DETAIL_ON - std::string _zw = "Targetimage: " + std::to_string((int) _Image->rgb_image) + " Size: " + std::to_string(_Image->width) + ", " + std::to_string(_Image->height); - _zw = _zw + " _zwImage: " + std::to_string((int) _zwImage.rgb_image) + " Size: " + std::to_string(_zwImage.width) + ", " + std::to_string(_zwImage.height); - LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, _zw); -#endif + #ifdef DEBUG_DETAIL_ON + std::string _zw = "Targetimage: " + std::to_string((int) _Image->rgb_image) + " Size: " + std::to_string(_Image->width) + ", " + std::to_string(_Image->height); + _zw = _zw + " _zwImage: " + std::to_string((int) _zwImage.rgb_image) + " Size: " + std::to_string(_zwImage.width) + ", " + std::to_string(_zwImage.height); + LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, _zw); + #endif for (int x = 0; x < width; ++x) for (int y = 0; y < height; ++y) { p_target = _Image->rgb_image + (channels * (y * width + x)); - p_source = _zwImage.rgb_image + (channels * (y * width + x)); + p_source = _zwImage->rgb_image + (channels * (y * width + x)); p_target[0] = p_source[0]; p_target[1] = p_source[1]; p_target[2] = p_source[2]; } -#ifdef DEBUG_DETAIL_ON - LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After Copy To Target"); -#endif + #ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After Copy To Target"); + #endif - free(buf); + delete _zwImage; -#ifdef DEBUG_DETAIL_ON - LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Done"); -#endif + #ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Done"); + #endif return ESP_OK; } @@ -376,21 +363,21 @@ esp_err_t CCamera::CaptureToFile(std::string nm, int delay) } LEDOnOff(false); -#ifdef DEBUG_DETAIL_ON - ESP_LOGD(TAG, "w %d, h %d, size %d", fb->width, fb->height, fb->len); -#endif + #ifdef DEBUG_DETAIL_ON + ESP_LOGD(TAG, "w %d, h %d, size %d", fb->width, fb->height, fb->len); + #endif nm = FormatFileName(nm); -#ifdef DEBUG_DETAIL_ON - ESP_LOGD(TAG, "Save Camera to: %s", nm.c_str()); -#endif + #ifdef DEBUG_DETAIL_ON + ESP_LOGD(TAG, "Save Camera to: %s", nm.c_str()); + #endif ftype = toUpper(getFileType(nm)); -#ifdef DEBUG_DETAIL_ON - ESP_LOGD(TAG, "Filetype: %s", ftype.c_str()); -#endif + #ifdef DEBUG_DETAIL_ON + ESP_LOGD(TAG, "Filetype: %s", ftype.c_str()); + #endif uint8_t * buf = NULL; size_t buf_len = 0; @@ -604,7 +591,7 @@ void CCamera::GetCameraParameter(httpd_req_t *req, int &qual, framesize_t &resol if (qual < 0) qual = 0; } - }; + } } @@ -674,6 +661,7 @@ bool CCamera::getCameraInitSuccessful() return CameraInitSuccessful; } + std::vector demoFiles; void CCamera::useDemoMode() diff --git a/code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp b/code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp index 923f4a3b..1b1646fe 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp @@ -1,6 +1,7 @@ #include "ClassFlowAlignment.h" #include "ClassFlowMakeImage.h" #include "ClassFlow.h" +#include "server_tflite.h" #include "CRotateImage.h" #include "esp_log.h" @@ -29,6 +30,9 @@ void ClassFlowAlignment::SetInitialParameter(void) AlignAndCutImage = NULL; ImageBasis = NULL; ImageTMP = NULL; + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG + AlgROI = (ImageData*)heap_caps_malloc(sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + #endif previousElement = NULL; disabled = false; SAD_criteria = 0.05; @@ -161,6 +165,25 @@ string ClassFlowAlignment::getHTMLSingleStep(string host) bool ClassFlowAlignment::doFlow(string time) { + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG + if (!AlgROI) // AlgROI needs to be allocated before ImageTMP to avoid heap fragmentation + { + AlgROI = (ImageData*)heap_caps_realloc(AlgROI, sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + if (!AlgROI) + { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate AlgROI"); + LogFile.WriteHeapInfo("ClassFlowAlignment-doFlow"); + tfliteflow.SetNewAlgROI(false); // continue flow only with alg.jpg (no ROIs available) + } + } + + if (AlgROI) + { + ImageBasis->writeToMemoryAsJPG((ImageData*)AlgROI, 90); + tfliteflow.SetNewAlgROI(true); + } + #endif + if (!ImageTMP) { ImageTMP = new CImageBasis(ImageBasis); @@ -214,8 +237,15 @@ bool ClassFlowAlignment::doFlow(string time) SaveReferenceAlignmentValues(); } - if (SaveAllFiles) AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/alg.jpg")); - + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG + if (AlgROI) { + DrawRef(ImageTMP); + tfliteflow.DigitalDrawROI(ImageTMP); + tfliteflow.AnalogDrawROI(ImageTMP); + ImageTMP->writeToMemoryAsJPG((ImageData*)AlgROI, 90); + } + #endif + if (SaveAllFiles) { if (initialflip) @@ -224,7 +254,8 @@ bool ClassFlowAlignment::doFlow(string time) ImageTMP->width = ImageTMP->height; ImageTMP->height = _zw; } - DrawRef(ImageTMP); + + AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/alg.jpg")); ImageTMP->SaveToFile(FormatFileName("/sdcard/img_tmp/alg_roi.jpg")); } diff --git a/code/components/jomjol_flowcontroll/ClassFlowAlignment.h b/code/components/jomjol_flowcontroll/ClassFlowAlignment.h index 29ce8cfd..6b7b7914 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowAlignment.h +++ b/code/components/jomjol_flowcontroll/ClassFlowAlignment.h @@ -34,6 +34,9 @@ protected: public: CImageBasis *ImageBasis, *ImageTMP; + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG + ImageData *AlgROI; + #endif ClassFlowAlignment(std::vector* lfc); diff --git a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp index 637f9c4e..06e8ea1e 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp @@ -141,6 +141,27 @@ t_CNNType ClassFlowControll::GetTypeAnalog() } +#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG +void ClassFlowControll::DigitalDrawROI(CImageBasis *_zw) +{ + if (flowdigit) + flowdigit->DrawROI(_zw); +} + + +void ClassFlowControll::AnalogDrawROI(CImageBasis *_zw) +{ + if (flowanalog) + flowanalog->DrawROI(_zw); +} + + +void ClassFlowControll::SetNewAlgROI(bool _value) +{ + bNewAlgROI = _value; +} +#endif + #ifdef ENABLE_MQTT string ClassFlowControll::GetMQTTMainTopic() @@ -667,25 +688,47 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req) } } else if (_fn == "alg_roi.jpg") { - _send = new CImageBasis(flowalignment->ImageBasis); - - if (_send->ImageOkay()) { - if (flowalignment) flowalignment->DrawRef(_send); - if (flowdigit) flowdigit->DrawROI(_send); - if (flowanalog) flowanalog->DrawROI(_send); - _sendDelete = true; // delete temporary _send element after sending - } - else { - LogFile.WriteToFile(ESP_LOG_WARN, TAG, "ClassFlowControll::GetJPGStream: Not enough memory to create alg_roi.jpg -> alg.jpg is going to be served!"); - - if (flowalignment && flowalignment->ImageBasis->ImageOkay()) { - _send = flowalignment->ImageBasis; + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG // no CImageBasis needed to create alg_roi.jpg (ca. 790kB less RAM) + if (bNewAlgROI) { + if (flowalignment && flowalignment->AlgROI) { + httpd_resp_set_type(req, "image/jpeg"); + result = httpd_resp_send(req, (const char *)flowalignment->AlgROI->data, flowalignment->AlgROI->size); + } + else { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "ClassFlowControll::GetJPGStream: alg_roi.jpg cannot be served"); + return ESP_FAIL; + } } else { - httpd_resp_send(req, NULL, 0); - return ESP_OK; + if (flowalignment && flowalignment->ImageBasis->ImageOkay()) { + _send = flowalignment->ImageBasis; + } + else { + httpd_resp_send(req, NULL, 0); + return ESP_OK; + } } - } + #else + _send = new CImageBasis(flowalignment->ImageBasis); + + if (_send->ImageOkay()) { + if (flowalignment) flowalignment->DrawRef(_send); + if (flowdigit) flowdigit->DrawROI(_send); + if (flowanalog) flowanalog->DrawROI(_send); + _sendDelete = true; // delete temporary _send element after sending + } + else { + LogFile.WriteToFile(ESP_LOG_WARN, TAG, "ClassFlowControll::GetJPGStream: Not enough memory to create alg_roi.jpg -> alg.jpg is going to be served!"); + + if (flowalignment && flowalignment->ImageBasis->ImageOkay()) { + _send = flowalignment->ImageBasis; + } + else { + httpd_resp_send(req, NULL, 0); + return ESP_OK; + } + } + #endif } else { std::vector htmlinfo; diff --git a/code/components/jomjol_flowcontroll/ClassFlowControll.h b/code/components/jomjol_flowcontroll/ClassFlowControll.h index a1013647..5bdeafb0 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowControll.h +++ b/code/components/jomjol_flowcontroll/ClassFlowControll.h @@ -38,6 +38,9 @@ protected: void SetInitialParameter(void); std::string aktstatus; int aktRunNr; + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG + bool bNewAlgROI = false; + #endif public: void InitFlow(std::string config); @@ -51,11 +54,20 @@ public: bool ReadParameter(FILE* pfile, string& aktparamgraph); string getJSON(); string getNumbersName(); + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG + void SetNewAlgROI(bool _value); + #endif string TranslateAktstatus(std::string _input); -#ifdef ENABLE_MQTT + + #ifdef ENABLE_MQTT string GetMQTTMainTopic(); -#endif //ENABLE_MQTT + #endif //ENABLE_MQTT + + #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG + void DigitalDrawROI(CImageBasis *_zw); + void AnalogDrawROI(CImageBasis *_zw); + #endif esp_err_t GetJPGStream(std::string _fn, httpd_req_t *req); esp_err_t SendRawJPG(httpd_req_t *req); @@ -71,9 +83,10 @@ public: t_CNNType GetTypeDigital(); t_CNNType GetTypeAnalog(); -#ifdef ENABLE_MQTT + + #ifdef ENABLE_MQTT bool StartMQTTService(); -#endif //ENABLE_MQTT + #endif //ENABLE_MQTT int CleanTempFolder(); diff --git a/code/components/jomjol_image_proc/CImageBasis.cpp b/code/components/jomjol_image_proc/CImageBasis.cpp index 86149e91..49f188e7 100644 --- a/code/components/jomjol_image_proc/CImageBasis.cpp +++ b/code/components/jomjol_image_proc/CImageBasis.cpp @@ -25,9 +25,9 @@ uint8_t * CImageBasis::RGBImageLock(int _waitmaxsec) { if (islocked) { -#ifdef DEBUG_DETAIL_ON - ESP_LOGD(TAG, "Image is locked: sleep for: %ds", _waitmaxsec); -#endif + #ifdef DEBUG_DETAIL_ON + ESP_LOGD(TAG, "Image is locked: sleep for: %ds", _waitmaxsec); + #endif TickType_t xDelay; xDelay = 1000 / portTICK_PERIOD_MS; for (int i = 0; i <= _waitmaxsec; ++i) @@ -84,6 +84,19 @@ ImageData* CImageBasis::writeToMemoryAsJPG(const int quality) } +void CImageBasis::writeToMemoryAsJPG(ImageData* i, const int quality) +{ + ImageData* ii = new ImageData; + + RGBImageLock(); + stbi_write_jpg_to_func(writejpghelp, ii, width, height, channels, rgb_image, quality); + RGBImageRelease(); + + memCopy((uint8_t*) ii, (uint8_t*) i, sizeof(ImageData)); + delete ii; +} + + struct SendJPGHTTP { httpd_req_t *req; @@ -380,6 +393,28 @@ void CImageBasis::CreateEmptyImage(int _width, int _height, int _channels) } +void CImageBasis::EmptyImage() +{ + #ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("CImageBasis::EmptyImage"); + #endif + + stbi_uc* p_source; + + RGBImageLock(); + + for (int x = 0; x < width; ++x) + for (int y = 0; y < height; ++y) + { + p_source = rgb_image + (channels * (y * width + x)); + for (int _channels = 0; _channels < channels; ++_channels) + p_source[_channels] = (uint8_t) 0; + } + + RGBImageRelease(); +} + + void CImageBasis::LoadFromMemory(stbi_uc *_buffer, int len) { RGBImageLock(); @@ -503,8 +538,8 @@ CImageBasis::CImageBasis(std::string _image) #ifdef DEBUG_DETAIL_ON std::string zw = "CImageBasis after load " + _image; - ESP_LOGD(TAG, "%s", zw.c_str()); - ESP_LOGD(TAG, "w %d, h %d, b %d, c %d", width, height, bpp, channels); + ESP_LOGD(TAG, "%s", zw.c_str()); + ESP_LOGD(TAG, "w %d, h %d, b %d, c %d", width, height, bpp, channels); #endif #ifdef DEBUG_DETAIL_ON diff --git a/code/components/jomjol_image_proc/CImageBasis.h b/code/components/jomjol_image_proc/CImageBasis.h index 4983251c..eddf1cfd 100644 --- a/code/components/jomjol_image_proc/CImageBasis.h +++ b/code/components/jomjol_image_proc/CImageBasis.h @@ -61,6 +61,7 @@ class CImageBasis void SetIndepended(){externalImage = false;}; void CreateEmptyImage(int _width, int _height, int _channels); + void EmptyImage(); CImageBasis(); @@ -74,7 +75,8 @@ class CImageBasis void LoadFromMemory(stbi_uc *_buffer, int len); - ImageData* writeToMemoryAsJPG(const int quality = 90); + ImageData* writeToMemoryAsJPG(const int quality = 90); + void writeToMemoryAsJPG(ImageData* ii, const int quality = 90); esp_err_t SendJPGtoHTTP(httpd_req_t *req, const int quality = 90); diff --git a/code/components/jomjol_tfliteclass/CTfLiteClass.cpp b/code/components/jomjol_tfliteclass/CTfLiteClass.cpp index 2dd6a364..5d4ac844 100644 --- a/code/components/jomjol_tfliteclass/CTfLiteClass.cpp +++ b/code/components/jomjol_tfliteclass/CTfLiteClass.cpp @@ -170,9 +170,9 @@ bool CTfLiteClass::LoadInputImageBasis(CImageBasis *rs) input_data_ptr++; } -#ifdef DEBUG_DETAIL_ON + #ifdef DEBUG_DETAIL_ON LogFile.WriteHeapInfo("CTfLiteClass::LoadInputImageBasis - done"); -#endif + #endif return true; } @@ -191,21 +191,21 @@ bool CTfLiteClass::MakeAllocate() if (this->interpreter) { - TfLiteStatus allocate_status = this->interpreter->AllocateTensors(); - if (allocate_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter, "AllocateTensors() failed"); - LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "AllocateTensors() failed"); + TfLiteStatus allocate_status = this->interpreter->AllocateTensors(); + if (allocate_status != kTfLiteOk) { + TF_LITE_REPORT_ERROR(error_reporter, "AllocateTensors() failed"); + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "AllocateTensors() failed"); - this->GetInputDimension(); + this->GetInputDimension(); return false; - } + } } else { LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "new tflite::MicroInterpreter failed"); LogFile.WriteHeapInfo("CTfLiteClass::MakeAllocate-new tflite::MicroInterpreter failed"); return false; -} + } #ifdef DEBUG_DETAIL_ON diff --git a/code/include/defines.h b/code/include/defines.h index 6a24f600..b7bf0508 100644 --- a/code/include/defines.h +++ b/code/include/defines.h @@ -88,6 +88,10 @@ #define READOUT_TYPE_RAWVALUE 2 #define READOUT_TYPE_ERROR 3 + //ClassFlowControll: Serve alg_roi.jpg from memory as JPG + #define ALGROI_LOAD_FROM_MEM_AS_JPG // Load ALG_ROI.JPG as rendered JPG from RAM + #define ALGROI_LOAD_FROM_MEM_AS_JPG__SHOW_TAKE_IMAGE_PROCESS // Show take image image processing on webinterface (overview.html) + //ClassFlowMQTT #define LWT_TOPIC "connection" #define LWT_CONNECTED "connected"