Mirror/AI-on-the-edge-device
1
0
Fork 0
mirror of https://github.com/jomjol/AI-on-the-edge-device.git synced 2025-12-08 12:36:52 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add error handling for memory intensive tasks (#1798)

Browse Source 
* tflite model loading: error handling

* FlowAlignment: error handling

* CImageBasis+GetJPGStream : error handling
This commit is contained in:
Slider0007
2023-01-11 20:23:26 +01:00
committed by GitHub
parent a39092497e
commit 58816275e5
8 changed files with 465 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp
View File

@@ -12,8 +12,6 @@
static const char *TAG = "ALIGN";
bool AlignmentExtendedDebugging = true;
// #define DEBUG_DETAIL_ON
@@ -36,6 +34,7 @@ void ClassFlowAlignment::SetInitialParameter(void)
SAD_criteria = 0.05;
}
ClassFlowAlignment::ClassFlowAlignment(std::vector<ClassFlow*>* lfc)
{
SetInitialParameter();
@@ -51,7 +50,7 @@ ClassFlowAlignment::ClassFlowAlignment(std::vector<ClassFlow*>* lfc)
if (!ImageBasis) // the function take pictures does not exist --> must be created first ONLY FOR TEST PURPOSES
{
if (AlignmentExtendedDebugging) ESP_LOGD(TAG, "CImageBasis had to be created");
ESP_LOGD(TAG, "CImageBasis had to be created");
ImageBasis = new CImageBasis(namerawimage);
}
}
@@ -148,6 +147,7 @@ bool ClassFlowAlignment::ReadParameter(FILE* pfile, string& aktparamgraph)
}
string ClassFlowAlignment::getHTMLSingleStep(string host)
{
string result;
@@ -162,11 +162,24 @@ string ClassFlowAlignment::getHTMLSingleStep(string host)
bool ClassFlowAlignment::doFlow(string time)
{
if (!ImageTMP)
{
ImageTMP = new CImageBasis(ImageBasis);
if (!ImageTMP)
{
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate ImageTMP -> Exec this round aborted!");
LogFile.WriteHeapInfo("ClassFlowAlignment-doFlow");
return false;
}
}
delete AlignAndCutImage;
AlignAndCutImage = new CAlignAndCutImage(ImageBasis, ImageTMP);
if (!AlignAndCutImage)
{
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate AlignAndCutImage -> Exec this round aborted!");
LogFile.WriteHeapInfo("ClassFlowAlignment-doFlow");
return false;
}
CRotateImage rt(AlignAndCutImage, ImageTMP, initialflip);
if (initialflip)
@@ -176,10 +189,13 @@ bool ClassFlowAlignment::doFlow(string time)
ImageBasis->width = _zw;
}
if (initialmirror){
if (initialmirror)
{
ESP_LOGD(TAG, "do mirror");
rt.Mirror();
if (SaveAllFiles) AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/mirror.jpg"));
if (SaveAllFiles)
AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/mirror.jpg"));
}
if ((initalrotate != 0) || initialflip)
@@ -188,7 +204,9 @@ bool ClassFlowAlignment::doFlow(string time)
rt.RotateAntiAliasing(initalrotate);
else
rt.Rotate(initalrotate);
if (SaveAllFiles) AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/rot.jpg"));
if (SaveAllFiles)
AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/rot.jpg"));
}
if (!AlignAndCutImage->Align(&References[0], &References[1]))
@@ -220,7 +238,6 @@ bool ClassFlowAlignment::doFlow(string time)
}
void ClassFlowAlignment::SaveReferenceAlignmentValues()
{
FILE* pFile;
@@ -260,9 +277,6 @@ void ClassFlowAlignment::SaveReferenceAlignmentValues()
}
bool ClassFlowAlignment::LoadReferenceAlignmentValues(void)
{
FILE* pFile;
@@ -326,10 +340,11 @@ bool ClassFlowAlignment::LoadReferenceAlignmentValues(void)
}
void ClassFlowAlignment::DrawRef(CImageBasis *_zw)
{
if (_zw->ImageOkay())
{
_zw->drawRect(References[0].target_x, References[0].target_y, References[0].width, References[0].height, 255, 0, 0, 2);
_zw->drawRect(References[1].target_x, References[1].target_y, References[1].width, References[1].height, 255, 0, 0, 2);
}
}

48
code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp
View File

@@ -12,6 +12,7 @@
static const char* TAG = "CNN";
//#define DEBUG_DETAIL_ON
ClassFlowCNNGeneral::ClassFlowCNNGeneral(ClassFlowAlignment *_flowalign, t_CNNType _cnntype) : ClassFlowImage(NULL, TAG)
@@ -31,6 +32,7 @@ ClassFlowCNNGeneral::ClassFlowCNNGeneral(ClassFlowAlignment *_flowalign, t_CNNTy
logfileRetentionInDays = 5;
}
string ClassFlowCNNGeneral::getReadout(int _analog = 0, bool _extendedResolution, int prev, float _before_narrow_Analog, float analogDigitalTransitionStart)
{
string result = "";
@@ -124,11 +126,10 @@ string ClassFlowCNNGeneral::getReadout(int _analog = 0, bool _extendedResolution
}
return result;
}
return result;
}
int ClassFlowCNNGeneral::PointerEvalHybridNew(float number, float number_of_predecessors, int eval_predecessors, bool Analog_Predecessors, float digitalAnalogTransitionStart)
{
int result;
@@ -245,6 +246,7 @@ int ClassFlowCNNGeneral::PointerEvalAnalogToDigitNew(float number, float numeral
}
int ClassFlowCNNGeneral::PointerEvalAnalogNew(float number, int numeral_preceder)
{
float number_min, number_max;
@@ -285,7 +287,6 @@ int ClassFlowCNNGeneral::PointerEvalAnalogNew(float number, int numeral_preceder
" number: " + std::to_string(number) + " numeral_preceder = " + std::to_string(numeral_preceder) + " Analog_error = " + std::to_string(Analog_error));
return result;
}
@@ -382,6 +383,7 @@ bool ClassFlowCNNGeneral::ReadParameter(FILE* pfile, string& aktparamgraph)
return true;
}
general* ClassFlowCNNGeneral::FindGENERAL(string _name_number)
{
for (int i = 0; i < GENERAL.size(); ++i)
@@ -434,7 +436,6 @@ general* ClassFlowCNNGeneral::GetGENERAL(string _name, bool _create = true)
}
string ClassFlowCNNGeneral::getHTMLSingleStep(string host)
{
string result, zw;
@@ -459,7 +460,6 @@ string ClassFlowCNNGeneral::getHTMLSingleStep(string host)
}
bool ClassFlowCNNGeneral::doFlow(string time)
{
if (disabled)
@@ -477,6 +477,7 @@ bool ClassFlowCNNGeneral::doFlow(string time)
return true;
}
bool ClassFlowCNNGeneral::doAlignAndCut(string time)
{
if (disabled)
@@ -511,7 +512,10 @@ bool ClassFlowCNNGeneral::doAlignAndCut(string time)
return true;
}
void ClassFlowCNNGeneral::DrawROI(CImageBasis *_zw)
{
if (_zw->ImageOkay())
{
if (CNNType == Analogue || CNNType == Analogue100)
{
@@ -535,6 +539,8 @@ void ClassFlowCNNGeneral::DrawROI(CImageBasis *_zw)
_zw->drawRect(GENERAL[_dig]->ROI[i]->posx, GENERAL[_dig]->ROI[i]->posy, GENERAL[_dig]->ROI[i]->deltax, GENERAL[_dig]->ROI[i]->deltay, 0, 0, (255 - _dig*100), 2);
}
}
}
bool ClassFlowCNNGeneral::getNetworkParameter()
{
@@ -546,11 +552,18 @@ bool ClassFlowCNNGeneral::getNetworkParameter()
zwcnn = FormatFileName(zwcnn);
ESP_LOGD(TAG, "%s", zwcnn.c_str());
if (!tflite->LoadModel(zwcnn)) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't read model file " + cnnmodelfile);
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't load tflite model " + cnnmodelfile + " -> Init aborted!");
LogFile.WriteHeapInfo("getNetworkParameter-LoadModel");
delete tflite;
return false;
}
if (!tflite->MakeAllocate()) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate tflite model -> Init aborted!");
LogFile.WriteHeapInfo("getNetworkParameter-MakeAllocate");
delete tflite;
return false;
}
tflite->MakeAllocate();
if (CNNType == AutoDetect)
{
@@ -601,6 +614,7 @@ bool ClassFlowCNNGeneral::getNetworkParameter()
return true;
}
bool ClassFlowCNNGeneral::doNeuralNetwork(string time)
{
if (disabled)
@@ -612,13 +626,20 @@ bool ClassFlowCNNGeneral::doNeuralNetwork(string time)
string zwcnn = "/sdcard" + cnnmodelfile;
zwcnn = FormatFileName(zwcnn);
ESP_LOGD(TAG, "%s", zwcnn.c_str());
if (!tflite->LoadModel(zwcnn)) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't read model file " + cnnmodelfile);
if (!tflite->LoadModel(zwcnn)) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't load tflite model " + cnnmodelfile + " -> Exec aborted this round!");
LogFile.WriteHeapInfo("doNeuralNetwork-LoadModel");
delete tflite;
return false;
}
if (!tflite->MakeAllocate()) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate tfilte model -> Exec aborted this round!");
LogFile.WriteHeapInfo("doNeuralNetwork-MakeAllocate");
delete tflite;
return false;
}
tflite->MakeAllocate();
for (int n = 0; n < GENERAL.size(); ++n) // For each NUMBER
{
@@ -805,6 +826,7 @@ bool ClassFlowCNNGeneral::doNeuralNetwork(string time)
return true;
}
bool ClassFlowCNNGeneral::isExtendedResolution(int _number)
{
if (!(CNNType == Digital))
@@ -814,7 +836,6 @@ bool ClassFlowCNNGeneral::isExtendedResolution(int _number)
}
std::vector<HTMLInfo*> ClassFlowCNNGeneral::GetHTMLInfo()
{
std::vector<HTMLInfo*> result;
@@ -856,11 +877,13 @@ std::vector<HTMLInfo*> ClassFlowCNNGeneral::GetHTMLInfo()
return result;
}
int ClassFlowCNNGeneral::getNumberGENERAL()
{
return GENERAL.size();
}
string ClassFlowCNNGeneral::getNameGENERAL(int _analog)
{
if (_analog < GENERAL.size())
@@ -869,6 +892,7 @@ string ClassFlowCNNGeneral::getNameGENERAL(int _analog)
return "GENERAL DOES NOT EXIST";
}
general* ClassFlowCNNGeneral::GetGENERAL(int _analog)
{
if (_analog < GENERAL.size())
@@ -878,7 +902,6 @@ general* ClassFlowCNNGeneral::GetGENERAL(int _analog)
}
void ClassFlowCNNGeneral::UpdateNameNumbers(std::vector<std::string> *_name_numbers)
{
for (int _dig = 0; _dig < GENERAL.size(); _dig++)
@@ -895,6 +918,7 @@ void ClassFlowCNNGeneral::UpdateNameNumbers(std::vector<std::string> *_name_numb
}
}
string ClassFlowCNNGeneral::getReadoutRawString(int _analog)
{
string rt = "";

107
code/components/jomjol_flowcontroll/ClassFlowControll.cpp
View File

@@ -29,6 +29,8 @@ extern "C" {
static const char* TAG = "CTRL";
//#define DEBUG_DETAIL_ON
std::string ClassFlowControll::doSingleStep(std::string _stepname, std::string _host){
std::string _classname = "";
@@ -110,6 +112,7 @@ std::vector<HTMLInfo*> ClassFlowControll::GetAllDigital()
return empty;
}
std::vector<HTMLInfo*> ClassFlowControll::GetAllAnalog()
{
if (flowanalog)
@@ -119,6 +122,7 @@ std::vector<HTMLInfo*> ClassFlowControll::GetAllAnalog()
return empty;
}
t_CNNType ClassFlowControll::GetTypeDigital()
{
if (flowdigit)
@@ -127,6 +131,7 @@ t_CNNType ClassFlowControll::GetTypeDigital()
return t_CNNType::None;
}
t_CNNType ClassFlowControll::GetTypeAnalog()
{
if (flowanalog)
@@ -158,6 +163,7 @@ bool ClassFlowControll::StartMQTTService() {
}
#endif //ENABLE_MQTT
void ClassFlowControll::SetInitialParameter(void)
{
AutoStart = false;
@@ -171,12 +177,14 @@ void ClassFlowControll::SetInitialParameter(void)
aktstatus = "Booting ...";
}
bool ClassFlowControll::isAutoStart(long &_intervall)
{
_intervall = AutoIntervall * 60 * 1000; // AutoInterval: minutes -> ms
return AutoStart;
}
ClassFlow* ClassFlowControll::CreateClassFlow(std::string _type)
{
ClassFlow* cfc = NULL;
@@ -238,6 +246,7 @@ ClassFlow* ClassFlowControll::CreateClassFlow(std::string _type)
return cfc;
}
void ClassFlowControll::InitFlow(std::string config)
{
string line;
@@ -279,14 +288,17 @@ void ClassFlowControll::InitFlow(std::string config)
}
fclose(pFile);
}
std::string* ClassFlowControll::getActStatus(){
std::string* ClassFlowControll::getActStatus()
{
return &aktstatus;
}
void ClassFlowControll::doFlowMakeImageOnly(string time){
void ClassFlowControll::doFlowMakeImageOnly(string time)
{
std::string zw_time;
for (int i = 0; i < FlowControll.size(); ++i)
@@ -304,6 +316,7 @@ void ClassFlowControll::doFlowMakeImageOnly(string time){
}
}
bool ClassFlowControll::doFlow(string time)
{
bool result = true;
@@ -332,8 +345,8 @@ bool ClassFlowControll::doFlow(string time)
MQTTPublish(mqttServer_getMainTopic() + "/" + "status", flowStatus, false);
#endif //ENABLE_MQTT
string zw = "FlowControll.doFlow - " + FlowControll[i]->name();
#ifdef DEBUG_DETAIL_ON
string zw = "FlowControll.doFlow - " + FlowControll[i]->name();
LogFile.WriteHeapInfo(zw);
#endif
@@ -356,8 +369,8 @@ bool ClassFlowControll::doFlow(string time)
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("ClassFlowControll::doFlow");
#endif
}
zw_time = getCurrentTimeString("%H:%M:%S");
std::string flowStatus = "Flow finished";
aktstatus = flowStatus + " (" + zw_time + ")";
@@ -365,6 +378,7 @@ bool ClassFlowControll::doFlow(string time)
#ifdef ENABLE_MQTT
MQTTPublish(mqttServer_getMainTopic() + "/" + "status", flowStatus, false);
#endif //ENABLE_MQTT
return result;
}
@@ -434,6 +448,7 @@ string ClassFlowControll::getReadout(bool _rawvalue = false, bool _noerror = fal
return result;
}
string ClassFlowControll::GetPrevalue(std::string _number)
{
if (flowpostprocessing)
@@ -444,6 +459,7 @@ string ClassFlowControll::GetPrevalue(std::string _number)
return std::string("");
}
std::string ClassFlowControll::UpdatePrevalue(std::string _newvalue, std::string _numbers, bool _extern)
{
float zw;
@@ -473,6 +489,7 @@ std::string ClassFlowControll::UpdatePrevalue(std::string _newvalue, std::string
return std::string();
}
bool ClassFlowControll::ReadParameter(FILE* pfile, string& aktparamgraph)
{
std::vector<string> splitted;
@@ -626,34 +643,53 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
{
ESP_LOGD(TAG, "ClassFlowControll::GetJPGStream %s", _fn.c_str());
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("ClassFlowControll::GetJPGStream - Start");
#endif
CImageBasis *_send = NULL;
esp_err_t result = ESP_FAIL;
bool Dodelete = false;
bool _sendDelete = false;
if (flowalignment == NULL)
{
ESP_LOGD(TAG, "Can't continue, flowalignment is NULL");
return ESP_FAIL;
ESP_LOGD(TAG, "ClassFloDControll::GetJPGStream: FloDalignment is not (yet) initialized. Interrupt serving!");
return ESP_OK;
}
if (_fn == "alg.jpg")
{
if (_fn == "alg.jpg") {
if (flowalignment && flowalignment->ImageBasis->ImageOkay()) {
_send = flowalignment->ImageBasis;
}
else
{
if (_fn == "alg_roi.jpg")
{
CImageBasis* _imgzw = new CImageBasis(flowalignment->ImageBasis);
flowalignment->DrawRef(_imgzw);
if (flowdigit) flowdigit->DrawROI(_imgzw);
if (flowanalog) flowanalog->DrawROI(_imgzw);
_send = _imgzw;
Dodelete = true;
else {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "ClassFlowControll::GetJPGStream: alg.jpg cannot be served");
return ESP_FAIL;
}
else
{
}
else if (_fn == "alg_roi.jpg") {
_send = new CImageBasis(flowalignment->ImageBasis);
if (_send->ImageOkay()) {
if (flowalignment) flowalignment->DrawRef(_send);
if (flowdigit) flowdigit->DrawROI(_send);
if (flowanalog) flowanalog->DrawROI(_send);
_sendDelete = true; // delete temporary _send element after sending
}
else {
LogFile.WriteToFile(ESP_LOG_WARN, TAG, "ClassFlowControll::GetJPGStream: Not enough memory to create alg_roi.jpg -> alg.jpg is going to be served!");
if (flowalignment && flowalignment->ImageBasis->ImageOkay()) {
_send = flowalignment->ImageBasis;
}
else {
httpd_resp_send(req, NULL, 0);
return ESP_OK;
}
}
}
else {
std::vector<HTMLInfo*> htmlinfo;
htmlinfo = GetAllDigital();
ESP_LOGD(TAG, "After getClassFlowControll::GetAllDigital");
@@ -664,6 +700,7 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
if (htmlinfo[i]->image)
_send = htmlinfo[i]->image;
}
if (_fn == htmlinfo[i]->filename_org)
{
if (htmlinfo[i]->image_org)
@@ -676,6 +713,8 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
if (!_send)
{
htmlinfo = GetAllAnalog();
ESP_LOGD(TAG, "After getClassFlowControll::GetAllAnalog");
for (int i = 0; i < htmlinfo.size(); ++i)
{
if (_fn == htmlinfo[i]->filename)
@@ -683,6 +722,7 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
if (htmlinfo[i]->image)
_send = htmlinfo[i]->image;
}
if (_fn == htmlinfo[i]->filename_org)
{
if (htmlinfo[i]->image_org)
@@ -691,25 +731,31 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
delete htmlinfo[i];
}
htmlinfo.clear();
}
}
}
}
}
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("ClassFlowControll::GetJPGStream - before send");
#endif
if (_send)
{
ESP_LOGD(TAG, "Sending file: %s ...", _fn.c_str());
set_content_type_from_file(req, _fn.c_str());
result = _send->SendJPGtoHTTP(req);
ESP_LOGD(TAG, "File sending complete");
/* Respond with an empty chunk to signal HTTP response completion */
httpd_resp_send_chunk(req, NULL, 0);
ESP_LOGD(TAG, "File sending complete");
if (_sendDelete)
delete _send;
_send = NULL;
}
if (Dodelete)
{
delete _send;
}
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("ClassFlowControll::GetJPGStream - done");
#endif
return result;
}
@@ -720,6 +766,7 @@ string ClassFlowControll::getNumbersName()
return flowpostprocessing->getNumbersName();
}
string ClassFlowControll::getJSON()
{
return flowpostprocessing->GetJSON();

4
code/components/jomjol_flowcontroll/ClassFlowMakeImage.cpp
View File

@@ -63,6 +63,7 @@ ClassFlowMakeImage::ClassFlowMakeImage(std::vector<ClassFlow*>* lfc) : ClassFlow
SetInitialParameter();
}
bool ClassFlowMakeImage::ReadParameter(FILE* pfile, string& aktparamgraph)
{
std::vector<string> splitted;
@@ -171,6 +172,7 @@ bool ClassFlowMakeImage::ReadParameter(FILE* pfile, string& aktparamgraph)
return true;
}
string ClassFlowMakeImage::getHTMLSingleStep(string host)
{
string result;
@@ -178,6 +180,7 @@ string ClassFlowMakeImage::getHTMLSingleStep(string host)
return result;
}
bool ClassFlowMakeImage::doFlow(string zwtime)
{
string logPath = CreateLogFolder(zwtime);
@@ -215,6 +218,7 @@ bool ClassFlowMakeImage::doFlow(string zwtime)
return true;
}
esp_err_t ClassFlowMakeImage::SendRawJPG(httpd_req_t *req)
{
int flash_duration = (int) (waitbeforepicture * 1000);

140
code/components/jomjol_image_proc/CImageBasis.cpp
View File

@@ -18,6 +18,9 @@ using namespace std;
static const char *TAG = "C IMG BASIS";
//#define DEBUG_DETAIL_ON
uint8_t * CImageBasis::RGBImageLock(int _waitmaxsec)
{
if (islocked)
@@ -41,18 +44,19 @@ uint8_t * CImageBasis::RGBImageLock(int _waitmaxsec)
return rgb_image;
}
void CImageBasis::RGBImageRelease()
{
islocked = false;
}
uint8_t * CImageBasis::RGBImageGet()
{
return rgb_image;
}
void writejpghelp(void *context, void *data, int size)
{
// ESP_LOGD(TAG, "Size all: %d, size %d", ((ImageData*)context)->size, size);
@@ -68,8 +72,6 @@ void writejpghelp(void *context, void *data, int size)
}
ImageData* CImageBasis::writeToMemoryAsJPG(const int quality)
{
ImageData* ii = new ImageData;
@@ -90,13 +92,13 @@ struct SendJPGHTTP
int size = 0;
};
inline void writejpgtohttphelp(void *context, void *data, int size)
{
SendJPGHTTP* _send = (SendJPGHTTP*) context;
if ((_send->size + size) >= HTTP_BUFFER_SENT) // data no longer fits in buffer
{
httpd_req_t *_req = _send->req;
if (httpd_resp_send_chunk(_req, _send->buf, _send->size) != ESP_OK)
if (httpd_resp_send_chunk(_send->req, _send->buf, _send->size) != ESP_OK)
{
ESP_LOGE(TAG, "File sending failed!");
_send->res = ESP_FAIL;
@@ -108,7 +110,6 @@ inline void writejpgtohttphelp(void *context, void *data, int size)
}
esp_err_t CImageBasis::SendJPGtoHTTP(httpd_req_t *_req, const int quality)
{
SendJPGHTTP ii;
@@ -118,7 +119,6 @@ esp_err_t CImageBasis::SendJPGtoHTTP(httpd_req_t *_req, const int quality)
RGBImageLock();
stbi_write_jpg_to_func(writejpgtohttphelp, &ii, width, height, channels, rgb_image, quality);
RGBImageRelease();
if (ii.size > 0)
{
@@ -129,17 +129,18 @@ esp_err_t CImageBasis::SendJPGtoHTTP(httpd_req_t *_req, const int quality)
}
}
RGBImageRelease();
return ii.res;
}
bool CImageBasis::CopyFromMemory(uint8_t* _source, int _size)
{
int gr = height * width * channels;
if (gr != _size) // Size does not fit
{
ESP_LOGD(TAG, "Cannot copy image from memory - sizes do not match: should be %d, but is %d", _size, gr);
ESP_LOGE(TAG, "Cannot copy image from memory - sizes do not match: should be %d, but is %d", _size, gr);
return false;
}
@@ -150,6 +151,7 @@ bool CImageBasis::CopyFromMemory(uint8_t* _source, int _size)
return true;
}
uint8_t CImageBasis::GetPixelColor(int x, int y, int ch)
{
stbi_uc* p_source;
@@ -168,6 +170,7 @@ void CImageBasis::memCopy(uint8_t* _source, uint8_t* _target, int _size)
#endif
}
bool CImageBasis::isInImage(int x, int y)
{
if ((x < 0) || (x > width - 1))
@@ -179,6 +182,7 @@ bool CImageBasis::isInImage(int x, int y)
return true;
}
void CImageBasis::setPixelColor(int x, int y, int r, int g, int b)
{
stbi_uc* p_source;
@@ -194,6 +198,7 @@ void CImageBasis::setPixelColor(int x, int y, int r, int g, int b)
RGBImageRelease();
}
void CImageBasis::drawRect(int x, int y, int dx, int dy, int r, int g, int b, int thickness)
{
int zwx1, zwx2, zwy1, zwy2;
@@ -203,6 +208,9 @@ void CImageBasis::drawRect(int x, int y, int dx, int dy, int r, int g, int b, in
zwx2 = x + dx + thickness - 1;
zwy1 = y;
zwy2 = y;
RGBImageLock();
for (_thick = 0; _thick < thickness; _thick++)
for (_x = zwx1; _x <= zwx2; ++_x)
for (_y = zwy1; _y <= zwy2; _y++)
@@ -239,14 +247,18 @@ void CImageBasis::drawRect(int x, int y, int dx, int dy, int r, int g, int b, in
if (isInImage(_x, _y))
setPixelColor(_x + _thick, _y, r, g, b);
RGBImageRelease();
}
void CImageBasis::drawLine(int x1, int y1, int x2, int y2, int r, int g, int b, int thickness)
{
int _x, _y, _thick;
int _zwy1, _zwy2;
thickness = (thickness-1) / 2;
RGBImageLock();
for (_thick = 0; _thick <= thickness; ++_thick)
for (_x = x1 - _thick; _x <= x2 + _thick; ++_x)
{
@@ -265,8 +277,11 @@ void CImageBasis::drawLine(int x1, int y1, int x2, int y2, int r, int g, int b,
if (isInImage(_x, _y))
setPixelColor(_x, _y, r, g, b);
}
RGBImageRelease();
}
void CImageBasis::drawEllipse(int x1, int y1, int radx, int rady, int r, int g, int b, int thickness)
{
float deltarad, aktrad;
@@ -278,6 +293,8 @@ void CImageBasis::drawEllipse(int x1, int y1, int radx, int rady, int r, int g,
deltarad = 1 / (4 * M_PI * (rad + thickness - 1));
RGBImageLock();
for (aktrad = 0; aktrad <= (2 * M_PI); aktrad += deltarad)
for (_thick = 0; _thick < thickness; ++_thick)
{
@@ -286,6 +303,8 @@ void CImageBasis::drawEllipse(int x1, int y1, int radx, int rady, int r, int g,
if (isInImage(_x, _y))
setPixelColor(_x, _y, r, g, b);
}
RGBImageRelease();
}
@@ -296,6 +315,8 @@ void CImageBasis::drawCircle(int x1, int y1, int rad, int r, int g, int b, int t
deltarad = 1 / (4 * M_PI * (rad + thickness - 1));
RGBImageLock();
for (aktrad = 0; aktrad <= (2 * M_PI); aktrad += deltarad)
for (_thick = 0; _thick < thickness; ++_thick)
{
@@ -304,8 +325,11 @@ void CImageBasis::drawCircle(int x1, int y1, int rad, int r, int g, int b, int t
if (isInImage(_x, _y))
setPixelColor(_x, _y, r, g, b);
}
RGBImageRelease();
}
CImageBasis::CImageBasis()
{
externalImage = false;
@@ -316,6 +340,7 @@ CImageBasis::CImageBasis()
islocked = false;
}
void CImageBasis::CreateEmptyImage(int _width, int _height, int _channels)
{
bpp = _channels;
@@ -325,10 +350,21 @@ void CImageBasis::CreateEmptyImage(int _width, int _height, int _channels)
RGBImageLock();
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CImageBasis::CreateEmptyImage");
#endif
int memsize = width * height * channels;
rgb_image = (unsigned char*)GET_MEMORY(memsize);
if (rgb_image == NULL)
{
//ESP_LOGE(TAG, "CImageBasis::CreateEmptyImage: No more free memory!! Needed: %d %d %d %d", width, height, channels, memsize);
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "CImageBasis::CreateEmptyImage: Can't allocate enough memory: " + std::to_string(memsize));
LogFile.WriteHeapInfo("CImageBasis::CreateEmptyImage");
RGBImageRelease();
return;
}
stbi_uc* p_source;
@@ -341,25 +377,27 @@ void CImageBasis::CreateEmptyImage(int _width, int _height, int _channels)
}
RGBImageRelease();
}
void CImageBasis::LoadFromMemory(stbi_uc *_buffer, int len)
{
RGBImageLock();
if (rgb_image)
stbi_image_free(rgb_image);
rgb_image = stbi_load_from_memory(_buffer, len, &width, &height, &channels, 3);
bpp = channels;
ESP_LOGD(TAG, "Image loaded from memory: %d, %d, %d", width, height, channels);
if ((width * height * channels) == 0)
{
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Image with size 0 loaded --> reboot to be done! "
"Check that your camera module is working and connected properly.");
LogFile.WriteHeapInfo("CImageBasis::LoadFromMemory");
doReboot();
}
RGBImageRelease();
}
@@ -376,19 +414,27 @@ CImageBasis::CImageBasis(CImageBasis *_copyfrom)
RGBImageLock();
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CImageBasis::CImageBasis_copyfrom - Start");
#endif
int memsize = width * height * channels;
rgb_image = (unsigned char*)GET_MEMORY(memsize);
if (!rgb_image)
if (rgb_image == NULL)
{
ESP_LOGD(TAG, "%s", getESPHeapInfo().c_str());
ESP_LOGD(TAG, "No more free memory!! Needed: %d %d %d %d", width, height, channels, memsize);
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "CImageBasis::CImageBasis-Copyfrom: Can't allocate enough memory: " + std::to_string(memsize));
LogFile.WriteHeapInfo("CImageBasis::CImageBasis-Copyfrom");
RGBImageRelease();
return;
}
memCopy(_copyfrom->rgb_image, rgb_image, memsize);
RGBImageRelease();
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CImageBasis::CImageBasis_copyfrom - done");
#endif
}
@@ -401,15 +447,28 @@ CImageBasis::CImageBasis(int _width, int _height, int _channels)
height = _height;
bpp = _channels;
int memsize = width * height * channels;
RGBImageLock();
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CImageBasis::CImageBasis_width,height,ch - Start");
#endif
int memsize = width * height * channels;
rgb_image = (unsigned char*)GET_MEMORY(memsize);
if (!rgb_image)
if (rgb_image == NULL)
{
ESP_LOGD(TAG, "%s", getESPHeapInfo().c_str());
ESP_LOGD(TAG, "No more free memory!! Needed: %d %d %d %d", width, height, channels, memsize);
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "CImageBasis::CImageBasis-width,height,ch: Can't allocate enough memory: " + std::to_string(memsize));
LogFile.WriteHeapInfo("CImageBasis::CImageBasis-width,height,ch");
RGBImageRelease();
return;
}
RGBImageRelease();
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CImageBasis::CImageBasis_width,height,ch - done");
#endif
}
@@ -419,8 +478,6 @@ CImageBasis::CImageBasis(std::string _image)
channels = 3;
externalImage = false;
filename = _image;
long zwld = esp_get_free_heap_size();
ESP_LOGD(TAG, "freeheapsize before: %ld", zwld);
if (file_size(_image.c_str()) == 0) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, _image + " is empty!");
@@ -428,32 +485,39 @@ CImageBasis::CImageBasis(std::string _image)
}
RGBImageLock();
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CImageBasis::CImageBasis_image - Start");
#endif
rgb_image = stbi_load(_image.c_str(), &width, &height, &bpp, channels);
if (rgb_image == NULL) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to load " + _image + "! Is it corrupted?");
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "CImageBasis::CImageBasis-image: Failed to load " + _image + "! Is it corrupted?");
LogFile.WriteHeapInfo("CImageBasis::CImageBasis-image");
RGBImageRelease();
return;
}
RGBImageRelease();
zwld = esp_get_free_heap_size();
ESP_LOGD(TAG, "freeheapsize after: %ld", zwld);
std::string zw = "Image Load failed:" + _image;
if (rgb_image == NULL)
ESP_LOGD(TAG, "%s", zw.c_str());
zw = "CImageBasis after load " + _image + "\n";
#ifdef DEBUG_DETAIL_ON
std::string zw = "CImageBasis after load " + _image;
ESP_LOGD(TAG, "%s", zw.c_str());
ESP_LOGD(TAG, "w %d, h %d, b %d, c %d", width, height, bpp, channels);
#endif
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CImageBasis::CImageBasis_image - done");
#endif
}
bool CImageBasis::ImageOkay(){
return rgb_image != NULL;
}
CImageBasis::CImageBasis(uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp)
{
islocked = false;
@@ -465,6 +529,7 @@ CImageBasis::CImageBasis(uint8_t* _rgb_image, int _channels, int _width, int _he
externalImage = true;
}
void CImageBasis::Contrast(float _contrast) //input range [-100..100]
{
stbi_uc* p_source;
@@ -486,13 +551,18 @@ void CImageBasis::Contrast(float _contrast) //input range [-100..100]
RGBImageRelease();
}
CImageBasis::~CImageBasis()
{
RGBImageLock();
if (!externalImage)
stbi_image_free(rgb_image);
RGBImageRelease();
}
void CImageBasis::SaveToFile(std::string _imageout)
{
string typ = getFileType(_imageout);
@@ -521,25 +591,24 @@ void CImageBasis::Resize(int _new_dx, int _new_dy)
RGBImageLock();
stbir_resize_uint8(rgb_image, width, height, 0, odata, _new_dx, _new_dy, 0, channels);
stbi_image_free(rgb_image);
rgb_image = (unsigned char*)GET_MEMORY(memsize);
memCopy(odata, rgb_image, memsize);
RGBImageRelease();
width = _new_dx;
height = _new_dy;
stbi_image_free(odata);
RGBImageRelease();
}
void CImageBasis::Resize(int _new_dx, int _new_dy, CImageBasis *_target)
{
if ((_target->height != _new_dy) || (_target->width != _new_dx) || (_target->channels != channels))
{
ESP_LOGD(TAG, "CImageBasis::Resize - Target image size does not fit!");
ESP_LOGE(TAG, "CImageBasis::Resize - Target image size does not fit!");
return;
}
@@ -547,6 +616,7 @@ void CImageBasis::Resize(int _new_dx, int _new_dy, CImageBasis *_target)
uint8_t* odata = _target->rgb_image;
stbir_resize_uint8(rgb_image, width, height, 0, odata, _new_dx, _new_dy, 0, channels);
RGBImageRelease();
}

125
code/components/jomjol_tfliteclass/CTfLiteClass.cpp
View File

@@ -74,6 +74,7 @@ int CTfLiteClass::GetOutClassification(int _von, int _bis)
return (zw_class - _von);
}
void CTfLiteClass::GetInputDimension(bool silent = false)
{
TfLiteTensor* input2 = this->interpreter->input(0);
@@ -92,6 +93,7 @@ void CTfLiteClass::GetInputDimension(bool silent = false)
}
}
int CTfLiteClass::ReadInputDimenstion(int _dim)
{
if (_dim == 0)
@@ -105,7 +107,6 @@ int CTfLiteClass::ReadInputDimenstion(int _dim)
}
int CTfLiteClass::GetAnzOutPut(bool silent)
{
TfLiteTensor* output2 = this->interpreter->output(0);
@@ -133,6 +134,7 @@ int CTfLiteClass::GetAnzOutPut(bool silent)
return numeroutput;
}
void CTfLiteClass::Invoke()
{
if (interpreter != nullptr)
@@ -140,10 +142,11 @@ void CTfLiteClass::Invoke()
}
bool CTfLiteClass::LoadInputImageBasis(CImageBasis *rs)
{
std::string zw = "ClassFlowCNNGeneral::doNeuralNetwork after LoadInputResizeImage: ";
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CTfLiteClass::LoadInputImageBasis - Start");
#endif
unsigned int w = rs->width;
unsigned int h = rs->height;
@@ -168,35 +171,53 @@ bool CTfLiteClass::LoadInputImageBasis(CImageBasis *rs)
}
#ifdef DEBUG_DETAIL_ON
LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "After loading in input");
LogFile.WriteHeapInfo("CTfLiteClass::LoadInputImageBasis - done");
#endif
return true;
}
void CTfLiteClass::MakeAllocate()
bool CTfLiteClass::MakeAllocate()
{
static tflite::AllOpsResolver resolver;
// ESP_LOGD(TAG, "%s", LogFile.getESPHeapInfo().c_str());
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CTLiteClass::Alloc start");
#endif
LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Make Allocate");
LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CTfLiteClass::MakeAllocate");
this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize, this->error_reporter);
// ESP_LOGD(TAG, "%s", LogFile.getESPHeapInfo().c_str());
if (this->interpreter)
{
TfLiteStatus allocate_status = this->interpreter->AllocateTensors();
if (allocate_status != kTfLiteOk) {
TF_LITE_REPORT_ERROR(error_reporter, "AllocateTensors() failed");
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "AllocateTensors() failed");
this->GetInputDimension();
return;
return false;
}
// ESP_LOGD(TAG, "Allocate Done");
}
else
{
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "new tflite::MicroInterpreter failed");
LogFile.WriteHeapInfo("CTfLiteClass::MakeAllocate-new tflite::MicroInterpreter failed");
return false;
}
void CTfLiteClass::GetInputTensorSize(){
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CTLiteClass::Alloc done");
#endif
return true;
}
void CTfLiteClass::GetInputTensorSize()
{
#ifdef DEBUG_DETAIL_ON
float *zw = this->input;
int test = sizeof(zw);
@@ -204,6 +225,7 @@ void CTfLiteClass::GetInputTensorSize(){
#endif
}
long CTfLiteClass::GetFileSize(std::string filename)
{
struct stat stat_buf;
@@ -212,90 +234,95 @@ long CTfLiteClass::GetFileSize(std::string filename)
}
unsigned char* CTfLiteClass::ReadFileToCharArray(std::string _fn)
bool CTfLiteClass::ReadFileToModel(std::string _fn)
{
long size;
LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CTfLiteClass::ReadFileToModel: " + _fn);
size = GetFileSize(_fn);
long size = GetFileSize(_fn);
if (size == -1)
{
ESP_LOGD(TAG, "File doesn't exist");
return NULL;
ESP_LOGE(TAG, "CTfLiteClass::ReadFileToModel: Model file doesn't exist: %s", _fn.c_str());
return false;
}
unsigned char *result = (unsigned char*) malloc(size);
int anz = 1;
while (!result && (anz < 6)) // Try a maximum of 5x (= 5s)
{
#ifdef DEBUG_DETAIL_ON
ESP_LOGD(TAG, "Speicher ist voll - Versuche es erneut: %d", anz);
LogFile.WriteHeapInfo("CTLiteClass::Alloc modelfile start");
#endif
result = (unsigned char*) malloc(size);
anz++;
}
modelfile = (unsigned char*)GET_MEMORY(size);
if(result != NULL) {
if(modelfile != NULL)
{
FILE* f = fopen(_fn.c_str(), "rb"); // previously only "r
fread(result, 1, size, f);
fread(modelfile, 1, size, f);
fclose(f);
}else {
ESP_LOGD(TAG, "No free memory available");
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("CTLiteClass::Alloc modelfile successful");
#endif
return true;
}
else
{
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "CTfLiteClass::ReadFileToModel: Can't allocate enough memory: " + std::to_string(size));
LogFile.WriteHeapInfo("CTfLiteClass::ReadFileToModel");
return false;
}
}
return result;
}
bool CTfLiteClass::LoadModel(std::string _fn){
bool CTfLiteClass::LoadModel(std::string _fn)
{
#ifdef SUPRESS_TFLITE_ERRORS
this->error_reporter = new tflite::OwnMicroErrorReporter;
#else
this->error_reporter = new tflite::MicroErrorReporter;
#endif
modelload = ReadFileToCharArray(_fn.c_str());
LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CTfLiteClass::LoadModel");
if (modelload == NULL)
if (!ReadFileToModel(_fn.c_str())) {
return false;
}
model = tflite::GetModel(modelload);
// free(rd);
TFLITE_MINIMAL_CHECK(model != nullptr);
model = tflite::GetModel(modelfile);
if(model == nullptr)
return false;
return true;
}
CTfLiteClass::CTfLiteClass()
{
this->model = nullptr;
this->modelfile = NULL;
this->interpreter = nullptr;
this->input = nullptr;
this->output = nullptr;
this->kTensorArenaSize = 800 * 1024; /// according to testfile: 108000 - so far 600;; 2021-09-11: 200 * 1024
this->tensor_arena = new uint8_t[kTensorArenaSize];
this->tensor_arena = (uint8_t*)GET_MEMORY(kTensorArenaSize);
}
CTfLiteClass::~CTfLiteClass()
{
delete this->tensor_arena;
free(modelfile);
free(this->tensor_arena);
delete this->interpreter;
delete this->error_reporter;
free(modelload);
}
namespace tflite {
int OwnMicroErrorReporter::Report(const char* format, va_list args) {
namespace tflite
{
int OwnMicroErrorReporter::Report(const char* format, va_list args)
{
return 0;
}
}

6
code/components/jomjol_tfliteclass/CTfLiteClass.h
View File

@@ -39,7 +39,7 @@ class CTfLiteClass
int kTensorArenaSize;
uint8_t *tensor_arena;
unsigned char *modelload = NULL;
unsigned char *modelfile = NULL;
float* input;
@@ -47,13 +47,13 @@ class CTfLiteClass
int im_height, im_width, im_channel;
long GetFileSize(std::string filename);
unsigned char* ReadFileToCharArray(std::string _fn);
bool ReadFileToModel(std::string _fn);
public:
CTfLiteClass();
~CTfLiteClass();
bool LoadModel(std::string _fn);
void MakeAllocate();
bool MakeAllocate();
void GetInputTensorSize();
bool LoadInputImageBasis(CImageBasis *rs);
void Invoke();

10
code/main/server_main.cpp
View File

@@ -316,21 +316,19 @@ esp_err_t img_tmp_virtual_handler(httpd_req_t *req)
filetosend = std::string(filename);
ESP_LOGD(TAG, "File to upload: %s", filetosend.c_str());
// Serve raw.jpg
if (filetosend == "raw.jpg")
{
return GetRawJPG(req);
}
esp_err_t zw = GetJPG(filetosend, req);
if (zw == ESP_OK)
// Serve alg.jpg, alg_roi.jpg or digital and analog ROIs
if (ESP_OK == GetJPG(filetosend, req))
return ESP_OK;
// File wird nicht intern bereit gestellt --> klassischer weg:
#ifdef DEBUG_DETAIL_ON
LogFile.WriteHeapInfo("img_tmp_virtual_handler - Done");
#endif
// File was not served already --> serve with img_tmp_handler
return img_tmp_handler(req);
}