Mirror/AI-on-the-edge-device
1
0
Fork 0
mirror of https://github.com/jomjol/AI-on-the-edge-device.git synced 2025-12-08 12:36:52 +03:00
Code Issues Packages Projects Releases Wiki Activity

Store preprocessed image with ROI to RAM (#1809)

Browse Source 
* tflite model loading: error handling

* FlowAlignment: error handling

* CImageBasis+GetJPGStream : error handling

* store preprocessed ALG_ROI.jpg to memory

* Update
This commit is contained in:
Slider0007
2023-01-13 22:05:18 +01:00
committed by GitHub
parent defbd60ccf
commit f15347598a
9 changed files with 214 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
code/components/jomjol_controlcamera/ClassControllCamera.cpp
View File

@@ -236,18 +236,18 @@ void CCamera::EnableAutoExposure(int flash_duration)
esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
{
string ftype;
int _size;
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Start");
#endif
uint8_t *zwischenspeicher = NULL;
_Image->EmptyImage(); //Delete previous stored raw image -> black image
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG__SHOW_TAKE_IMAGE_PROCESS
tfliteflow.SetNewAlgROI(false);
#endif
LEDOnOff(true);
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Start");
#endif
if (delay > 0)
{
LightOnOff(true);
@@ -255,18 +255,18 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
vTaskDelay( xDelay );
}
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LightOn");
#endif
#endif
camera_fb_t * fb = esp_camera_fb_get();
esp_camera_fb_return(fb);
fb = esp_camera_fb_get();
if (!fb) {
ESP_LOGE(TAG, "CaptureToBasisImage: Capture Failed");
LEDOnOff(false);
LightOnOff(false);
ESP_LOGE(TAG, "CaptureToBasisImage: Capture Failed");
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "is not working anymore (CCamera::CaptureToBasisImage) - most probably caused by a hardware problem (instablility, ...). "
"System will reboot.");
doReboot();
@@ -279,20 +279,13 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
loadNextDemoImage(fb);
}
_size = fb->len;
zwischenspeicher = (uint8_t*) malloc(_size);
if (!zwischenspeicher)
{
ESP_LOGE(TAG, "Insufficient memory space for image in function CaptureToBasisImage()");
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Insufficient memory space for image in function CaptureToBasisImage()");
}
for (int i = 0; i < _size; ++i)
*(zwischenspeicher + i) = *(fb->buf + i);
CImageBasis* _zwImage = new CImageBasis();
_zwImage->LoadFromMemory(fb->buf, fb->len);
esp_camera_fb_return(fb);
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After fb_get");
#endif
#endif
LEDOnOff(false);
@@ -302,15 +295,9 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
// TickType_t xDelay = 1000 / portTICK_PERIOD_MS;
// vTaskDelay( xDelay ); // wait for power to recover
uint8_t * buf = NULL;
CImageBasis _zwImage;
_zwImage.LoadFromMemory(zwischenspeicher, _size);
free(zwischenspeicher);
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LoadFromMemory");
#endif
#endif
stbi_uc* p_target;
stbi_uc* p_source;
@@ -318,31 +305,31 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
int width = image_width;
int height = image_height;
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
std::string _zw = "Targetimage: " + std::to_string((int) _Image->rgb_image) + " Size: " + std::to_string(_Image->width) + ", " + std::to_string(_Image->height);
_zw = _zw + " _zwImage: " + std::to_string((int) _zwImage.rgb_image) + " Size: " + std::to_string(_zwImage.width) + ", " + std::to_string(_zwImage.height);
LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, _zw);
#endif
#endif
for (int x = 0; x < width; ++x)
for (int y = 0; y < height; ++y)
{
p_target = _Image->rgb_image + (channels * (y * width + x));
p_source = _zwImage.rgb_image + (channels * (y * width + x));
p_source = _zwImage->rgb_image + (channels * (y * width + x));
p_target[0] = p_source[0];
p_target[1] = p_source[1];
p_target[2] = p_source[2];
}
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After Copy To Target");
#endif
#endif
free(buf);
delete _zwImage;
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Done");
#endif
#endif
return ESP_OK;
}
@@ -376,21 +363,21 @@ esp_err_t CCamera::CaptureToFile(std::string nm, int delay)
}
LEDOnOff(false);
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
ESP_LOGD(TAG, "w %d, h %d, size %d", fb->width, fb->height, fb->len);
#endif
#endif
nm = FormatFileName(nm);
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
ESP_LOGD(TAG, "Save Camera to: %s", nm.c_str());
#endif
#endif
ftype = toUpper(getFileType(nm));
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
ESP_LOGD(TAG, "Filetype: %s", ftype.c_str());
#endif
#endif
uint8_t * buf = NULL;
size_t buf_len = 0;
@@ -604,7 +591,7 @@ void CCamera::GetCameraParameter(httpd_req_t *req, int &qual, framesize_t &resol
if (qual < 0)
qual = 0;
}
};
}
}
@@ -674,6 +661,7 @@ bool CCamera::getCameraInitSuccessful()
return CameraInitSuccessful;
}
std::vector<std::string> demoFiles;
void CCamera::useDemoMode()

35
code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp
View File

@@ -1,6 +1,7 @@
#include "ClassFlowAlignment.h"
#include "ClassFlowMakeImage.h"
#include "ClassFlow.h"
#include "server_tflite.h"
#include "CRotateImage.h"
#include "esp_log.h"
@@ -29,6 +30,9 @@ void ClassFlowAlignment::SetInitialParameter(void)
AlignAndCutImage = NULL;
ImageBasis = NULL;
ImageTMP = NULL;
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
AlgROI = (ImageData*)heap_caps_malloc(sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);
#endif
previousElement = NULL;
disabled = false;
SAD_criteria = 0.05;
@@ -161,6 +165,25 @@ string ClassFlowAlignment::getHTMLSingleStep(string host)
bool ClassFlowAlignment::doFlow(string time)
{
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
if (!AlgROI) // AlgROI needs to be allocated before ImageTMP to avoid heap fragmentation
{
AlgROI = (ImageData*)heap_caps_realloc(AlgROI, sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);
if (!AlgROI)
{
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate AlgROI");
LogFile.WriteHeapInfo("ClassFlowAlignment-doFlow");
tfliteflow.SetNewAlgROI(false); // continue flow only with alg.jpg (no ROIs available)
}
}
if (AlgROI)
{
ImageBasis->writeToMemoryAsJPG((ImageData*)AlgROI, 90);
tfliteflow.SetNewAlgROI(true);
}
#endif
if (!ImageTMP)
{
ImageTMP = new CImageBasis(ImageBasis);
@@ -214,7 +237,14 @@ bool ClassFlowAlignment::doFlow(string time)
SaveReferenceAlignmentValues();
}
if (SaveAllFiles) AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/alg.jpg"));
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
if (AlgROI) {
DrawRef(ImageTMP);
tfliteflow.DigitalDrawROI(ImageTMP);
tfliteflow.AnalogDrawROI(ImageTMP);
ImageTMP->writeToMemoryAsJPG((ImageData*)AlgROI, 90);
}
#endif
if (SaveAllFiles)
{
@@ -224,7 +254,8 @@ bool ClassFlowAlignment::doFlow(string time)
ImageTMP->width = ImageTMP->height;
ImageTMP->height = _zw;
}
DrawRef(ImageTMP);
AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/alg.jpg"));
ImageTMP->SaveToFile(FormatFileName("/sdcard/img_tmp/alg_roi.jpg"));
}

3
code/components/jomjol_flowcontroll/ClassFlowAlignment.h
View File

@@ -34,6 +34,9 @@ protected:
public:
CImageBasis *ImageBasis, *ImageTMP;
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
ImageData *AlgROI;
#endif
ClassFlowAlignment(std::vector<ClassFlow*>* lfc);

43
code/components/jomjol_flowcontroll/ClassFlowControll.cpp
View File

@@ -141,6 +141,27 @@ t_CNNType ClassFlowControll::GetTypeAnalog()
}
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
void ClassFlowControll::DigitalDrawROI(CImageBasis *_zw)
{
if (flowdigit)
flowdigit->DrawROI(_zw);
}
void ClassFlowControll::AnalogDrawROI(CImageBasis *_zw)
{
if (flowanalog)
flowanalog->DrawROI(_zw);
}
void ClassFlowControll::SetNewAlgROI(bool _value)
{
bNewAlgROI = _value;
}
#endif
#ifdef ENABLE_MQTT
string ClassFlowControll::GetMQTTMainTopic()
@@ -667,6 +688,27 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
}
}
else if (_fn == "alg_roi.jpg") {
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG // no CImageBasis needed to create alg_roi.jpg (ca. 790kB less RAM)
if (bNewAlgROI) {
if (flowalignment && flowalignment->AlgROI) {
httpd_resp_set_type(req, "image/jpeg");
result = httpd_resp_send(req, (const char *)flowalignment->AlgROI->data, flowalignment->AlgROI->size);
}
else {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "ClassFlowControll::GetJPGStream: alg_roi.jpg cannot be served");
return ESP_FAIL;
}
}
else {
if (flowalignment && flowalignment->ImageBasis->ImageOkay()) {
_send = flowalignment->ImageBasis;
}
else {
httpd_resp_send(req, NULL, 0);
return ESP_OK;
}
}
#else
_send = new CImageBasis(flowalignment->ImageBasis);
if (_send->ImageOkay()) {
@@ -686,6 +728,7 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
return ESP_OK;
}
}
#endif
}
else {
std::vector<HTMLInfo*> htmlinfo;

21
code/components/jomjol_flowcontroll/ClassFlowControll.h
View File

@@ -38,6 +38,9 @@ protected:
void SetInitialParameter(void);
std::string aktstatus;
int aktRunNr;
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
bool bNewAlgROI = false;
#endif
public:
void InitFlow(std::string config);
@@ -51,11 +54,20 @@ public:
bool ReadParameter(FILE* pfile, string& aktparamgraph);
string getJSON();
string getNumbersName();
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
void SetNewAlgROI(bool _value);
#endif
string TranslateAktstatus(std::string _input);
#ifdef ENABLE_MQTT
#ifdef ENABLE_MQTT
string GetMQTTMainTopic();
#endif //ENABLE_MQTT
#endif //ENABLE_MQTT
#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
void DigitalDrawROI(CImageBasis *_zw);
void AnalogDrawROI(CImageBasis *_zw);
#endif
esp_err_t GetJPGStream(std::string _fn, httpd_req_t *req);
esp_err_t SendRawJPG(httpd_req_t *req);
@@ -71,9 +83,10 @@ public:
t_CNNType GetTypeDigital();
t_CNNType GetTypeAnalog();
#ifdef ENABLE_MQTT
#ifdef ENABLE_MQTT
bool StartMQTTService();
#endif //ENABLE_MQTT
#endif //ENABLE_MQTT
int CleanTempFolder();

39
code/components/jomjol_image_proc/CImageBasis.cpp
View File

@@ -25,9 +25,9 @@ uint8_t * CImageBasis::RGBImageLock(int _waitmaxsec)
{
if (islocked)
{
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
ESP_LOGD(TAG, "Image is locked: sleep for: %ds", _waitmaxsec);
#endif
#endif
TickType_t xDelay;
xDelay = 1000 / portTICK_PERIOD_MS;
for (int i = 0; i <= _waitmaxsec; ++i)
@@ -84,6 +84,19 @@ ImageData* CImageBasis::writeToMemoryAsJPG(const int quality)
}
void CImageBasis::writeToMemoryAsJPG(ImageData* i, const int quality)
{
ImageData* ii = new ImageData;
RGBImageLock();
stbi_write_jpg_to_func(writejpghelp, ii, width, height, channels, rgb_image, quality);
RGBImageRelease();
memCopy((uint8_t*) ii, (uint8_t*) i, sizeof(ImageData));
delete ii;
}
struct SendJPGHTTP
{
httpd_req_t *req;
@@ -380,6 +393,28 @@ void CImageBasis::CreateEmptyImage(int _width, int _height, int _channels)
}
void CImageBasis::EmptyImage()
{
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CImageBasis::EmptyImage");
#endif
stbi_uc* p_source;
RGBImageLock();
for (int x = 0; x < width; ++x)
for (int y = 0; y < height; ++y)
{
p_source = rgb_image + (channels * (y * width + x));
for (int _channels = 0; _channels < channels; ++_channels)
p_source[_channels] = (uint8_t) 0;
}
RGBImageRelease();
}
void CImageBasis::LoadFromMemory(stbi_uc *_buffer, int len)
{
RGBImageLock();

2
code/components/jomjol_image_proc/CImageBasis.h
View File

@@ -61,6 +61,7 @@ class CImageBasis
void SetIndepended(){externalImage = false;};
void CreateEmptyImage(int _width, int _height, int _channels);
void EmptyImage();
CImageBasis();
@@ -75,6 +76,7 @@ class CImageBasis
void LoadFromMemory(stbi_uc *_buffer, int len);
ImageData* writeToMemoryAsJPG(const int quality = 90);
void writeToMemoryAsJPG(ImageData* ii, const int quality = 90);
esp_err_t SendJPGtoHTTP(httpd_req_t *req, const int quality = 90);

6
code/components/jomjol_tfliteclass/CTfLiteClass.cpp
View File

@@ -170,9 +170,9 @@ bool CTfLiteClass::LoadInputImageBasis(CImageBasis *rs)
input_data_ptr++;
}
#ifdef DEBUG_DETAIL_ON
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CTfLiteClass::LoadInputImageBasis - done");
#endif
#endif
return true;
}
@@ -205,7 +205,7 @@ bool CTfLiteClass::MakeAllocate()
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "new tflite::MicroInterpreter failed");
LogFile.WriteHeapInfo("CTfLiteClass::MakeAllocate-new tflite::MicroInterpreter failed");
return false;
}
}
#ifdef DEBUG_DETAIL_ON

4
code/include/defines.h
View File

@@ -88,6 +88,10 @@
#define READOUT_TYPE_RAWVALUE 2
#define READOUT_TYPE_ERROR 3
//ClassFlowControll: Serve alg_roi.jpg from memory as JPG
#define ALGROI_LOAD_FROM_MEM_AS_JPG // Load ALG_ROI.JPG as rendered JPG from RAM
#define ALGROI_LOAD_FROM_MEM_AS_JPG__SHOW_TAKE_IMAGE_PROCESS // Show take image image processing on webinterface (overview.html)
//ClassFlowMQTT
#define LWT_TOPIC "connection"
#define LWT_CONNECTED "connected"