diff --git a/Changelog.md b/Changelog.md index 00c233e5..bbc4a41f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,21 +1,17 @@ -## [unreleased] +## [15.1.1] - 2023-03-23 ### Update Procedure Update Procedure see [online documentation](https://jomjol.github.io/AI-on-the-edge-device-docs/Installation/#update-ota-over-the-air) -:bangbang: Afterwards you should force-reload the Web Interface (usually Ctrl-F5 will do it)! - -:bangbang: Afterwards you should check your configuration for errors! - ### Changes -For a full list of changes see [Full list of changes](https://github.com/jomjol/AI-on-the-edge-device/compare/v15.0.3...v15.1.0) +For a full list of changes see [Full list of changes](https://github.com/jomjol/AI-on-the-edge-device/compare/v15.1.0...v15.1.1) #### Added -- n.a. - +- [#2206](https://github.com/jomjol/AI-on-the-edge-device/pull/2206) Log PSRAM usage +- [#2216](https://github.com/jomjol/AI-on-the-edge-device/pull/2216) Log MQTT connection refused reasons #### Changed @@ -23,7 +19,10 @@ For a full list of changes see [Full list of changes](https://github.com/jomjol/ #### Fixed +- [#2224](https://github.com/jomjol/AI-on-the-edge-device/pull/2224), [#2213](https://github.com/jomjol/AI-on-the-edge-device/pull/2213) Reverted some of the PSRAM usage changes due to negative sideffects - [#2203](https://github.com/jomjol/AI-on-the-edge-device/issues/2203) Correct API for pure InfluxDB v1 +- [#2180](https://github.com/jomjol/AI-on-the-edge-device/pull/2180) Fixed links in Parameter Documentation +- Various minor fixes #### Removed @@ -934,6 +933,7 @@ External Illumination - Initial Version +[15.1.1]: https://github.com/jomjol/AI-on-the-edge-device/compare/v15.1.0...v15.1.1 [15.1.0]: https://github.com/jomjol/AI-on-the-edge-device/compare/v15.0.3...v15.1.0 [15.0.3]: https://github.com/jomjol/AI-on-the-edge-device/compare/v14.0.3...v15.0.3 [14.0.3]: https://github.com/jomjol/AI-on-the-edge-device/compare/v13.0.8...v14.0.3 diff --git a/code/components/jomjol_controlcamera/ClassControllCamera.cpp b/code/components/jomjol_controlcamera/ClassControllCamera.cpp index 13dd7ab3..8058fbf0 100644 --- a/code/components/jomjol_controlcamera/ClassControllCamera.cpp +++ b/code/components/jomjol_controlcamera/ClassControllCamera.cpp @@ -311,7 +311,7 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay) loadNextDemoImage(fb); } - CImageBasis* _zwImage = new CImageBasis(); + CImageBasis* _zwImage = new CImageBasis("zwImage"); if (_zwImage) { _zwImage->LoadFromMemory(fb->buf, fb->len); } diff --git a/code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp b/code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp index c27b7674..a8ffcfbc 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp @@ -8,6 +8,7 @@ #include "ClassLogFile.h" +#include "psram.h" #include "../../include/defines.h" @@ -31,7 +32,7 @@ void ClassFlowAlignment::SetInitialParameter(void) ImageBasis = NULL; ImageTMP = NULL; #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG - AlgROI = (ImageData*)heap_caps_malloc(sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + AlgROI = (ImageData*)malloc_psram_heap(std::string(TAG) + "->AlgROI", sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); #endif previousElement = NULL; disabled = false; @@ -55,7 +56,7 @@ ClassFlowAlignment::ClassFlowAlignment(std::vector* lfc) if (!ImageBasis) // the function take pictures does not exist --> must be created first ONLY FOR TEST PURPOSES { ESP_LOGD(TAG, "CImageBasis had to be created"); - ImageBasis = new CImageBasis(namerawimage); + ImageBasis = new CImageBasis("ImageBasis", namerawimage); } } @@ -189,7 +190,7 @@ bool ClassFlowAlignment::doFlow(string time) if (!ImageTMP) { - ImageTMP = new CImageBasis(ImageBasis); + ImageTMP = new CImageBasis("ImageTMP", ImageBasis); if (!ImageTMP) { LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate ImageTMP -> Exec this round aborted!"); @@ -199,7 +200,7 @@ bool ClassFlowAlignment::doFlow(string time) } delete AlignAndCutImage; - AlignAndCutImage = new CAlignAndCutImage(ImageBasis, ImageTMP); + AlignAndCutImage = new CAlignAndCutImage("AlignAndCutImage", ImageBasis, ImageTMP); if (!AlignAndCutImage) { LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate AlignAndCutImage -> Exec this round aborted!"); @@ -207,7 +208,7 @@ bool ClassFlowAlignment::doFlow(string time) return false; } - CRotateImage rt(AlignAndCutImage, ImageTMP, initialflip); + CRotateImage rt("rawImage", AlignAndCutImage, ImageTMP, initialflip); if (initialflip) { int _zw = ImageBasis->height; diff --git a/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp b/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp index ed071fa9..fb3800a1 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp @@ -381,8 +381,10 @@ bool ClassFlowCNNGeneral::ReadParameter(FILE* pfile, string& aktparamgraph) for (int _ana = 0; _ana < GENERAL.size(); ++_ana) for (int i = 0; i < GENERAL[_ana]->ROI.size(); ++i) { - GENERAL[_ana]->ROI[i]->image = new CImageBasis(modelxsize, modelysize, modelchannel); - GENERAL[_ana]->ROI[i]->image_org = new CImageBasis(GENERAL[_ana]->ROI[i]->deltax, GENERAL[_ana]->ROI[i]->deltay, 3); + GENERAL[_ana]->ROI[i]->image = new CImageBasis("ROI " + GENERAL[_ana]->ROI[i]->name, + modelxsize, modelysize, modelchannel); + GENERAL[_ana]->ROI[i]->image_org = new CImageBasis("ROI " + GENERAL[_ana]->ROI[i]->name + " original", + GENERAL[_ana]->ROI[i]->deltax, GENERAL[_ana]->ROI[i]->deltay, 3); } return true; diff --git a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp index d5995b70..e9d56e3e 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp @@ -826,7 +826,7 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req) return ESP_FAIL; } - _send = new CImageBasis(flowalignment->ImageBasis); + _send = new CImageBasis("alg_roi", flowalignment->ImageBasis); if (_send->ImageOkay()) { if (flowalignment) flowalignment->DrawRef(_send); diff --git a/code/components/jomjol_flowcontroll/ClassFlowTakeImage.cpp b/code/components/jomjol_flowcontroll/ClassFlowTakeImage.cpp index 5c19fb0b..bb8584af 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowTakeImage.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowTakeImage.cpp @@ -154,7 +154,7 @@ bool ClassFlowTakeImage::ReadParameter(FILE* pfile, string& aktparamgraph) image_width = Camera.image_width; image_height = Camera.image_height; - rawImage = new CImageBasis(); + rawImage = new CImageBasis("rawImage"); rawImage->CreateEmptyImage(image_width, image_height, 3); waitbeforepicture_store = waitbeforepicture; @@ -231,7 +231,7 @@ esp_err_t ClassFlowTakeImage::SendRawJPG(httpd_req_t *req) ImageData* ClassFlowTakeImage::SendRawImage() { - CImageBasis *zw = new CImageBasis(rawImage); + CImageBasis *zw = new CImageBasis("SendRawImage", rawImage); ImageData *id; int flash_duration = (int) (waitbeforepicture * 1000); Camera.CaptureToBasisImage(zw, flash_duration); diff --git a/code/components/jomjol_helper/Helper.cpp b/code/components/jomjol_helper/Helper.cpp index 3417eb58..ae3ab683 100644 --- a/code/components/jomjol_helper/Helper.cpp +++ b/code/components/jomjol_helper/Helper.cpp @@ -64,14 +64,14 @@ string getESPHeapInfo(){ sprintf(aMsgBuf," | SPI Free: %ld", (long) aFreeSPIHeapSize); espInfoResultStr += string(aMsgBuf); - sprintf(aMsgBuf," | SPI Larg Block: %ld", (long) aHeapLargestFreeBlockSize); + sprintf(aMsgBuf," | SPI Large Block: %ld", (long) aHeapLargestFreeBlockSize); espInfoResultStr += string(aMsgBuf); sprintf(aMsgBuf," | SPI Min Free: %ld", (long) aMinFreeHeapSize); espInfoResultStr += string(aMsgBuf); sprintf(aMsgBuf," | Int Free: %ld", (long) (aFreeInternalHeapSize)); espInfoResultStr += string(aMsgBuf); - sprintf(aMsgBuf," | Int Larg Block: %ld", (long) aHeapIntLargestFreeBlockSize); + sprintf(aMsgBuf," | Int Large Block: %ld", (long) aHeapIntLargestFreeBlockSize); espInfoResultStr += string(aMsgBuf); sprintf(aMsgBuf," | Int Min Free: %ld", (long) (aMinFreeInternalHeapSize)); espInfoResultStr += string(aMsgBuf); diff --git a/code/components/jomjol_helper/psram.cpp b/code/components/jomjol_helper/psram.cpp new file mode 100644 index 00000000..8cbb64a5 --- /dev/null +++ b/code/components/jomjol_helper/psram.cpp @@ -0,0 +1,42 @@ +#include "ClassLogFile.h" +#include "esp_heap_caps.h" + +static const char* TAG = "PSRAM"; + +using namespace std; + + +void *malloc_psram_heap(std::string name, size_t size, uint32_t caps) { + void *ptr; + + ptr = heap_caps_malloc(size, caps); + if (ptr != NULL) { + LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Allocated " + to_string(size) + " bytes in PSRAM for '" + name + "'"); + } + else { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to allocate " + to_string(size) + " bytes in PSRAM for '" + name + "'!"); + } + + return ptr; +} + + +void *calloc_psram_heap(std::string name, size_t n, size_t size, uint32_t caps) { + void *ptr; + + ptr = heap_caps_calloc(n, size, caps); + if (ptr != NULL) { + LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Allocated " + to_string(size) + " bytes in PSRAM for '" + name + "'"); + } + else { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to allocate " + to_string(size) + " bytes in PSRAM for '" + name + "'!"); + } + + return ptr; +} + + +void free_psram_heap(std::string name, void *ptr) { + LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Freeing memory in PSRAM used for '" + name + "'..."); + heap_caps_free(ptr); +} \ No newline at end of file diff --git a/code/components/jomjol_helper/psram.h b/code/components/jomjol_helper/psram.h new file mode 100644 index 00000000..fafd0c73 --- /dev/null +++ b/code/components/jomjol_helper/psram.h @@ -0,0 +1,7 @@ + +#include "esp_heap_caps.h" + +void *malloc_psram_heap(std::string name, size_t size, uint32_t caps); +void *calloc_psram_heap(std::string name, size_t n, size_t size, uint32_t caps); + +void free_psram_heap(std::string name, void *ptr); \ No newline at end of file diff --git a/code/components/jomjol_image_proc/CAlignAndCutImage.cpp b/code/components/jomjol_image_proc/CAlignAndCutImage.cpp index 86cbae6c..b7f7afbd 100644 --- a/code/components/jomjol_image_proc/CAlignAndCutImage.cpp +++ b/code/components/jomjol_image_proc/CAlignAndCutImage.cpp @@ -5,12 +5,14 @@ #include #include #include +#include "psram.h" #include "../../include/defines.h" static const char* TAG = "c_align_and_cut_image"; -CAlignAndCutImage::CAlignAndCutImage(CImageBasis *_org, CImageBasis *_temp) +CAlignAndCutImage::CAlignAndCutImage(std::string _name, CImageBasis *_org, CImageBasis *_temp) : CImageBasis(_name) { + name = _name; rgb_image = _org->rgb_image; channels = _org->channels; width = _org->width; @@ -37,7 +39,7 @@ bool CAlignAndCutImage::Align(RefInfo *_temp1, RefInfo *_temp2) int r0_x, r0_y, r1_x, r1_y; bool isSimilar1, isSimilar2; - CFindTemplate* ft = new CFindTemplate(rgb_image, channels, width, height, bpp); + CFindTemplate* ft = new CFindTemplate("align", rgb_image, channels, width, height, bpp); r0_x = _temp1->target_x; r0_y = _temp1->target_y; @@ -81,7 +83,7 @@ bool CAlignAndCutImage::Align(RefInfo *_temp1, RefInfo *_temp2) LogFile.WriteToDedicatedFile("/sdcard/alignment.txt", zw); #endif*/ - CRotateImage rt(this, ImageTMP); + CRotateImage rt("Align", this, ImageTMP); rt.Translate(dx, dy); rt.Rotate(d_winkel, _temp1->target_x, _temp1->target_y); ESP_LOGD(TAG, "Alignment: dx %d - dy %d - rot %f", dx, dy, d_winkel); @@ -107,7 +109,7 @@ void CAlignAndCutImage::CutAndSave(std::string _template1, int x1, int y1, int d dy = y2 - y1; int memsize = dx * dy * channels; - uint8_t* odata = (unsigned char*) GET_MEMORY(memsize); + uint8_t* odata = (unsigned char*) malloc_psram_heap(std::string(TAG) + "->odata", memsize, MALLOC_CAP_SPIRAM); stbi_uc* p_target; stbi_uc* p_source; @@ -186,7 +188,7 @@ CImageBasis* CAlignAndCutImage::CutAndSave(int x1, int y1, int dx, int dy) dy = y2 - y1; int memsize = dx * dy * channels; - uint8_t* odata = (unsigned char*)GET_MEMORY(memsize); + uint8_t* odata = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->odata", memsize, MALLOC_CAP_SPIRAM); stbi_uc* p_target; stbi_uc* p_source; @@ -202,7 +204,7 @@ CImageBasis* CAlignAndCutImage::CutAndSave(int x1, int y1, int dx, int dy) p_target[_channels] = p_source[_channels]; } - CImageBasis* rs = new CImageBasis(odata, channels, dx, dy, bpp); + CImageBasis* rs = new CImageBasis("CutAndSave", odata, channels, dx, dy, bpp); RGBImageRelease(); rs->SetIndepended(); return rs; diff --git a/code/components/jomjol_image_proc/CAlignAndCutImage.h b/code/components/jomjol_image_proc/CAlignAndCutImage.h index 7267cce3..46f5016e 100644 --- a/code/components/jomjol_image_proc/CAlignAndCutImage.h +++ b/code/components/jomjol_image_proc/CAlignAndCutImage.h @@ -12,9 +12,9 @@ class CAlignAndCutImage : public CImageBasis public: int t0_dx, t0_dy, t1_dx, t1_dy; CImageBasis *ImageTMP; - CAlignAndCutImage(std::string _image) : CImageBasis(_image) {ImageTMP = NULL;}; - CAlignAndCutImage(uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp) : CImageBasis(_rgb_image, _channels, _width, _height, _bpp) {ImageTMP = NULL;}; - CAlignAndCutImage(CImageBasis *_org, CImageBasis *_temp); + CAlignAndCutImage(std::string name, std::string _image) : CImageBasis(name, _image) {ImageTMP = NULL;}; + CAlignAndCutImage(std::string name, uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp) : CImageBasis(name, _rgb_image, _channels, _width, _height, _bpp) {ImageTMP = NULL;}; + CAlignAndCutImage(std::string name, CImageBasis *_org, CImageBasis *_temp); bool Align(RefInfo *_temp1, RefInfo *_temp2); // void Align(std::string _template1, int x1, int y1, std::string _template2, int x2, int y2, int deltax = 40, int deltay = 40, std::string imageROI = ""); diff --git a/code/components/jomjol_image_proc/CFindTemplate.h b/code/components/jomjol_image_proc/CFindTemplate.h index b281f502..14b0261b 100644 --- a/code/components/jomjol_image_proc/CFindTemplate.h +++ b/code/components/jomjol_image_proc/CFindTemplate.h @@ -32,7 +32,7 @@ class CFindTemplate : public CImageBasis { public: int tpl_width, tpl_height, tpl_bpp; - CFindTemplate(uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp) : CImageBasis(_rgb_image, _channels, _width, _height, _bpp) {}; + CFindTemplate(std::string name, uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp) : CImageBasis(name, _rgb_image, _channels, _width, _height, _bpp) {}; bool FindTemplate(RefInfo *_ref); diff --git a/code/components/jomjol_image_proc/CImageBasis.cpp b/code/components/jomjol_image_proc/CImageBasis.cpp index a8637e3f..ad6fee9d 100644 --- a/code/components/jomjol_image_proc/CImageBasis.cpp +++ b/code/components/jomjol_image_proc/CImageBasis.cpp @@ -1,5 +1,6 @@ #include "CImageBasis.h" #include "Helper.h" +#include "psram.h" #include "ClassLogFile.h" #include "server_ota.h" @@ -360,8 +361,9 @@ void CImageBasis::drawCircle(int x1, int y1, int rad, int r, int g, int b, int t } -CImageBasis::CImageBasis() +CImageBasis::CImageBasis(string _name) { + name = _name; externalImage = false; rgb_image = NULL; width = 0; @@ -384,8 +386,9 @@ void CImageBasis::CreateEmptyImage(int _width, int _height, int _channels) LogFile.WriteHeapInfo("CreateEmptyImage"); #endif - int memsize = width * height * channels; - rgb_image = (unsigned char*)GET_MEMORY(memsize); + memsize = width * height * channels; + + rgb_image = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->CImageBasis (" + name + ")", memsize, MALLOC_CAP_SPIRAM); if (rgb_image == NULL) { @@ -435,8 +438,10 @@ void CImageBasis::LoadFromMemory(stbi_uc *_buffer, int len) { RGBImageLock(); - if (rgb_image) + if (rgb_image != NULL) { stbi_image_free(rgb_image); + //free_psram_heap(std::string(TAG) + "->rgb_image (LoadFromMemory)", rgb_image); + } rgb_image = stbi_load_from_memory(_buffer, len, &width, &height, &channels, 3); bpp = channels; @@ -454,8 +459,9 @@ void CImageBasis::LoadFromMemory(stbi_uc *_buffer, int len) } -CImageBasis::CImageBasis(CImageBasis *_copyfrom) +CImageBasis::CImageBasis(string _name, CImageBasis *_copyfrom) { + name = _name; islocked = false; externalImage = false; channels = _copyfrom->channels; @@ -469,8 +475,9 @@ CImageBasis::CImageBasis(CImageBasis *_copyfrom) LogFile.WriteHeapInfo("CImageBasis_copyfrom - Start"); #endif - int memsize = width * height * channels; - rgb_image = (unsigned char*)GET_MEMORY(memsize); + memsize = width * height * channels; + + rgb_image = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->CImageBasis (" + name + ")", memsize, MALLOC_CAP_SPIRAM); if (rgb_image == NULL) { @@ -489,8 +496,9 @@ CImageBasis::CImageBasis(CImageBasis *_copyfrom) } -CImageBasis::CImageBasis(int _width, int _height, int _channels) +CImageBasis::CImageBasis(string _name, int _width, int _height, int _channels) { + name = _name; islocked = false; externalImage = false; channels = _channels; @@ -504,8 +512,9 @@ CImageBasis::CImageBasis(int _width, int _height, int _channels) LogFile.WriteHeapInfo("CImageBasis_width,height,ch - Start"); #endif - int memsize = width * height * channels; - rgb_image = (unsigned char*)GET_MEMORY(memsize); + memsize = width * height * channels; + + rgb_image = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->CImageBasis (" + name + ")", memsize, MALLOC_CAP_SPIRAM); if (rgb_image == NULL) { @@ -523,8 +532,9 @@ CImageBasis::CImageBasis(int _width, int _height, int _channels) } -CImageBasis::CImageBasis(std::string _image) +CImageBasis::CImageBasis(string _name, std::string _image) { + name = _name; islocked = false; channels = 3; externalImage = false; @@ -569,8 +579,9 @@ bool CImageBasis::ImageOkay(){ } -CImageBasis::CImageBasis(uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp) +CImageBasis::CImageBasis(string _name, uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp) { + name = _name; islocked = false; rgb_image = _rgb_image; channels = _channels; @@ -607,8 +618,10 @@ CImageBasis::~CImageBasis() { RGBImageLock(); - if (!externalImage) - stbi_image_free(rgb_image); + if (!externalImage) { + //stbi_image_free(rgb_image); + free_psram_heap(std::string(TAG) + "->CImageBasis (" + name + ", " + to_string(memsize) + ")", rgb_image); + } RGBImageRelease(); } @@ -637,19 +650,19 @@ void CImageBasis::SaveToFile(std::string _imageout) void CImageBasis::Resize(int _new_dx, int _new_dy) { - int memsize = _new_dx * _new_dy * channels; - uint8_t* odata = (unsigned char*)GET_MEMORY(memsize); + memsize = _new_dx * _new_dy * channels; + uint8_t* odata = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->odata", memsize, MALLOC_CAP_SPIRAM); RGBImageLock(); stbir_resize_uint8(rgb_image, width, height, 0, odata, _new_dx, _new_dy, 0, channels); - stbi_image_free(rgb_image); - rgb_image = (unsigned char*)GET_MEMORY(memsize); + rgb_image = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->CImageBasis Resize (" + name + ")", memsize, MALLOC_CAP_SPIRAM); memCopy(odata, rgb_image, memsize); width = _new_dx; height = _new_dy; - stbi_image_free(odata); + + free_psram_heap(std::string(TAG) + "->odata", odata); RGBImageRelease(); } diff --git a/code/components/jomjol_image_proc/CImageBasis.h b/code/components/jomjol_image_proc/CImageBasis.h index eddf1cfd..3f9c4b7f 100644 --- a/code/components/jomjol_image_proc/CImageBasis.h +++ b/code/components/jomjol_image_proc/CImageBasis.h @@ -30,6 +30,8 @@ class CImageBasis protected: bool externalImage; std::string filename; + std::string name; // Just used for diagnostics + int memsize = 0; void memCopy(uint8_t* _source, uint8_t* _target, int _size); bool isInImage(int x, int y); @@ -37,7 +39,7 @@ class CImageBasis bool islocked; public: - uint8_t* rgb_image; + uint8_t* rgb_image = NULL; int channels; int width, height, bpp; @@ -64,11 +66,11 @@ class CImageBasis void EmptyImage(); - CImageBasis(); - CImageBasis(std::string _image); - CImageBasis(uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp); - CImageBasis(int _width, int _height, int _channels); - CImageBasis(CImageBasis *_copyfrom); + CImageBasis(std::string name); + CImageBasis(std::string name, std::string _image); + CImageBasis(std::string name, uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp); + CImageBasis(std::string name, int _width, int _height, int _channels); + CImageBasis(std::string name, CImageBasis *_copyfrom); void Resize(int _new_dx, int _new_dy); void Resize(int _new_dx, int _new_dy, CImageBasis *_target); diff --git a/code/components/jomjol_image_proc/CRotateImage.cpp b/code/components/jomjol_image_proc/CRotateImage.cpp index 821c4269..12d44154 100644 --- a/code/components/jomjol_image_proc/CRotateImage.cpp +++ b/code/components/jomjol_image_proc/CRotateImage.cpp @@ -1,7 +1,10 @@ +#include #include "CRotateImage.h" +#include "psram.h" +static const char *TAG = "C ROTATE IMG"; -CRotateImage::CRotateImage(CImageBasis *_org, CImageBasis *_temp, bool _flip) +CRotateImage::CRotateImage(std::string _name, CImageBasis *_org, CImageBasis *_temp, bool _flip) : CImageBasis(_name) { rgb_image = _org->rgb_image; channels = _org->channels; @@ -15,6 +18,7 @@ CRotateImage::CRotateImage(CImageBasis *_org, CImageBasis *_temp, bool _flip) doflip = _flip; } + void CRotateImage::Mirror(){ int memsize = width * height * channels; uint8_t* odata; @@ -24,7 +28,7 @@ void CRotateImage::Mirror(){ } else { - odata = (unsigned char*)GET_MEMORY(memsize); + odata = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->odata", memsize, MALLOC_CAP_SPIRAM); } @@ -50,7 +54,7 @@ void CRotateImage::Mirror(){ // memcpy(rgb_image, odata, memsize); memCopy(odata, rgb_image, memsize); if (!ImageTMP) - stbi_image_free(odata); + free_psram_heap(std::string(TAG) + "->odata", odata); if (ImageTMP) ImageTMP->RGBImageRelease(); @@ -109,7 +113,7 @@ void CRotateImage::Rotate(float _angle, int _centerx, int _centery) } else { - odata = (unsigned char*)GET_MEMORY(memsize); + odata = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->odata", memsize, MALLOC_CAP_SPIRAM); } @@ -148,7 +152,7 @@ void CRotateImage::Rotate(float _angle, int _centerx, int _centery) if (!ImageTMP) { - stbi_image_free(odata); + free_psram_heap(std::string(TAG) + "->odata", odata); } if (ImageTMP) ImageTMP->RGBImageRelease(); @@ -209,7 +213,7 @@ void CRotateImage::RotateAntiAliasing(float _angle, int _centerx, int _centery) } else { - odata = (unsigned char*)GET_MEMORY(memsize); + odata = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->odata", memsize, MALLOC_CAP_SPIRAM); } @@ -269,7 +273,7 @@ void CRotateImage::RotateAntiAliasing(float _angle, int _centerx, int _centery) if (!ImageTMP) { - stbi_image_free(odata); + free_psram_heap(std::string(TAG) + "->odata", odata); } if (ImageTMP) ImageTMP->RGBImageRelease(); @@ -300,7 +304,7 @@ void CRotateImage::Translate(int _dx, int _dy) } else { - odata = (unsigned char*)GET_MEMORY(memsize); + odata = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->odata", memsize, MALLOC_CAP_SPIRAM); } @@ -336,7 +340,7 @@ void CRotateImage::Translate(int _dx, int _dy) memCopy(odata, rgb_image, memsize); if (!ImageTMP) { - stbi_image_free(odata); + free_psram_heap(std::string(TAG) + "->odata", odata); } if (ImageTMP) diff --git a/code/components/jomjol_image_proc/CRotateImage.h b/code/components/jomjol_image_proc/CRotateImage.h index f4113f5e..30c758d3 100644 --- a/code/components/jomjol_image_proc/CRotateImage.h +++ b/code/components/jomjol_image_proc/CRotateImage.h @@ -8,12 +8,13 @@ class CRotateImage: public CImageBasis { + public: CImageBasis *ImageTMP, *ImageOrg; bool doflip; - CRotateImage(std::string _image, bool _flip = false) : CImageBasis(_image) {ImageTMP = NULL; ImageOrg = NULL; doflip = _flip;}; - CRotateImage(uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp, bool _flip = false) : CImageBasis(_rgb_image, _channels, _width, _height, _bpp) {ImageTMP = NULL; ImageOrg = NULL; doflip = _flip;}; - CRotateImage(CImageBasis *_org, CImageBasis *_temp, bool _flip = false); + CRotateImage(std::string name, std::string _image, bool _flip = false) : CImageBasis(name, _image) {ImageTMP = NULL; ImageOrg = NULL; doflip = _flip;}; + CRotateImage(std::string name, uint8_t* _rgb_image, int _channels, int _width, int _height, int _bpp, bool _flip = false) : CImageBasis(name, _rgb_image, _channels, _width, _height, _bpp) {ImageTMP = NULL; ImageOrg = NULL; doflip = _flip;}; + CRotateImage(std::string name, CImageBasis *_org, CImageBasis *_temp, bool _flip = false); void Rotate(float _angle); void RotateAntiAliasing(float _angle); diff --git a/code/components/jomjol_mqtt/interface_mqtt.cpp b/code/components/jomjol_mqtt/interface_mqtt.cpp index 359cd348..7ce4bb3e 100644 --- a/code/components/jomjol_mqtt/interface_mqtt.cpp +++ b/code/components/jomjol_mqtt/interface_mqtt.cpp @@ -139,6 +139,31 @@ static esp_err_t mqtt_event_handler_cb(esp_mqtt_event_handle_t event) { break; case MQTT_EVENT_ERROR: + // http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718033 --> chapter 3.2.2.3 + + // The server does not support the level of the MQTT protocol requested by the client + // NOTE: Only protocol 3.1.1 is supported (refer to setting in sdkconfig) + if (event->error_handle->connect_return_code == MQTT_CONNECTION_REFUSE_PROTOCOL) { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Connection refused, unacceptable protocol version (0x01)"); + } + // The client identifier is correct UTF-8 but not allowed by the server + // e.g. clientID empty (cannot be the case -> default set in firmware) + else if (event->error_handle->connect_return_code == MQTT_CONNECTION_REFUSE_ID_REJECTED) { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Connection refused, identifier rejected (0x02)"); + } + // The network connection has been made but the MQTT service is unavailable + else if (event->error_handle->connect_return_code == MQTT_CONNECTION_REFUSE_SERVER_UNAVAILABLE) { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Connection refused, Server unavailable (0x03)"); + } + // The data in the user name or password is malformed + else if (event->error_handle->connect_return_code == MQTT_CONNECTION_REFUSE_BAD_USERNAME) { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Connection refused, malformed data in username or password (0x04)"); + } + // The client is not authorized to connect + else if (event->error_handle->connect_return_code == MQTT_CONNECTION_REFUSE_NOT_AUTHORIZED) { + LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Connection refused, not authorized. Check username/password (0x05)"); + } + #ifdef DEBUG_DETAIL_ON ESP_LOGD(TAG, "MQTT_EVENT_ERROR - esp_mqtt_error_codes:"); ESP_LOGD(TAG, "error_type:%d", event->error_handle->error_type); @@ -148,7 +173,7 @@ static esp_err_t mqtt_event_handler_cb(esp_mqtt_event_handle_t event) { ESP_LOGD(TAG, "esp_tls_stack_err:%d", event->error_handle->esp_tls_stack_err); ESP_LOGD(TAG, "esp_tls_cert_verify_flags:%d", event->error_handle->esp_tls_cert_verify_flags); #endif - //mqtt_connected = false; + break; default: @@ -235,7 +260,6 @@ int MQTT_Init() { .reconnect_timeout_ms = 15000, // Try to reconnect to broker (Default: 10000ms) .network_timeout_ms = 20000, // Network Timeout (Default: 10000ms) .message_retransmit_timeout = 3000 // Time after message resent when broker not acknowledged (QoS1, QoS2) - }; if (user.length() && password.length()){ diff --git a/code/components/jomjol_mqtt/mqtt_outbox.c b/code/components/jomjol_mqtt/mqtt_outbox.c index 498ad7d6..c001a8b2 100644 --- a/code/components/jomjol_mqtt/mqtt_outbox.c +++ b/code/components/jomjol_mqtt/mqtt_outbox.c @@ -8,6 +8,10 @@ #include "esp_log.h" #include "esp_heap_caps.h" +/* Enable this to use the PSRAM for MQTT Publishing. + * This saves 10 kBytes of RAM, see https://github.com/jomjol/AI-on-the-edge-device/pull/2113 + * However we can run into PSRAM fragmentation issues, leading to insufficient large blocks to load the model. + * See https://github.com/jomjol/AI-on-the-edge-device/issues/2200 */ #define USE_PSRAM #ifdef CONFIG_MQTT_CUSTOM_OUTBOX diff --git a/code/components/jomjol_tfliteclass/CTfLiteClass.cpp b/code/components/jomjol_tfliteclass/CTfLiteClass.cpp index 5d4ac844..6a2d4711 100644 --- a/code/components/jomjol_tfliteclass/CTfLiteClass.cpp +++ b/code/components/jomjol_tfliteclass/CTfLiteClass.cpp @@ -1,6 +1,7 @@ #include "CTfLiteClass.h" #include "ClassLogFile.h" #include "Helper.h" +#include "psram.h" #include "esp_log.h" #include "../../include/defines.h" @@ -250,7 +251,7 @@ bool CTfLiteClass::ReadFileToModel(std::string _fn) LogFile.WriteHeapInfo("CTLiteClass::Alloc modelfile start"); #endif - modelfile = (unsigned char*)GET_MEMORY(size); + modelfile = (unsigned char*)malloc_psram_heap(std::string(TAG) + "->modelfile", size, MALLOC_CAP_SPIRAM); if(modelfile != NULL) { @@ -305,17 +306,17 @@ CTfLiteClass::CTfLiteClass() this->input = nullptr; this->output = nullptr; this->kTensorArenaSize = 800 * 1024; /// according to testfile: 108000 - so far 600;; 2021-09-11: 200 * 1024 - this->tensor_arena = (uint8_t*)GET_MEMORY(kTensorArenaSize); + this->tensor_arena = (uint8_t*)malloc_psram_heap(std::string(TAG) + "->tensor_arena", kTensorArenaSize, MALLOC_CAP_SPIRAM); } CTfLiteClass::~CTfLiteClass() { - free(modelfile); - - free(this->tensor_arena); delete this->interpreter; delete this->error_reporter; + + free_psram_heap(std::string(TAG) + "->modelfile", modelfile); + free_psram_heap(std::string(TAG) + "->tensor_arena", this->tensor_arena); } diff --git a/code/components/jomjol_tfliteclass/server_tflite.cpp b/code/components/jomjol_tfliteclass/server_tflite.cpp index c22a30c2..68ed4d89 100644 --- a/code/components/jomjol_tfliteclass/server_tflite.cpp +++ b/code/components/jomjol_tfliteclass/server_tflite.cpp @@ -25,6 +25,7 @@ #include "read_wlanini.h" #include "connect_wlan.h" +#include "psram.h" ClassFlowControll tfliteflow; @@ -151,12 +152,12 @@ esp_err_t handler_get_heap(httpd_req_t *req) std::string zw = "Heap info:
" + getESPHeapInfo(); #ifdef TASK_ANALYSIS_ON - char* pcTaskList = (char*) heap_caps_calloc(1, sizeof(char) * 768, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + char* pcTaskList = (char*) calloc_psram_heap(std::string(TAG) + "->pcTaskList", 1, sizeof(char) * 768, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); if (pcTaskList) { vTaskList(pcTaskList); zw = zw + "

Task info:
Name | State | Prio | Lowest stacksize | Creation order | CPU (-1=NoAffinity)
"
                     + std::string(pcTaskList) + "
"; - heap_caps_free(pcTaskList); + free_psram_heap(std::string(TAG) + "->pcTaskList", pcTaskList); } else { zw = zw + "

Task info:
ERROR - Allocation of TaskList buffer in PSRAM failed"; @@ -617,11 +618,11 @@ esp_err_t handler_editflow(httpd_req_t *req) string out2 = out.substr(0, out.length() - 4) + "_org.jpg"; - CAlignAndCutImage *caic = new CAlignAndCutImage(in); + CAlignAndCutImage *caic = new CAlignAndCutImage("cutref", in); caic->CutAndSave(out2, x, y, dx, dy); delete caic; - CImageBasis *cim = new CImageBasis(out2); + CImageBasis *cim = new CImageBasis("cutref", out2); if (enhance) { cim->Contrast(90); diff --git a/code/include/defines.h b/code/include/defines.h index 7dc83127..a64aeee0 100644 --- a/code/include/defines.h +++ b/code/include/defines.h @@ -131,14 +131,8 @@ //CImageBasis #define HTTP_BUFFER_SENT 1024 - #define GET_MEMORY(X) heap_caps_malloc(X, MALLOC_CAP_SPIRAM) #define MAX_JPG_SIZE 128000 - - //CAlignAndCutImage + CImageBasis - #define _USE_MATH_DEFINES - #define GET_MEMORY(X) heap_caps_malloc(X, MALLOC_CAP_SPIRAM) - //make_stb + stb_image_resize + stb_image_write + stb_image //do not work if not in make_stb.cpp //#define STB_IMAGE_IMPLEMENTATION //#define STB_IMAGE_WRITE_IMPLEMENTATION diff --git a/code/sdkconfig.defaults b/code/sdkconfig.defaults index d2056178..9d15610f 100644 --- a/code/sdkconfig.defaults +++ b/code/sdkconfig.defaults @@ -1,7 +1,7 @@ ################################################## # Application specific configuration # Edit this file instead of sdkconfig.esp32cam! -# After editting, make sure to explicitly delete +# After editing make sure to explicitly delete # sdkconfig.esp32cam to apply your changes! ################################################## @@ -94,13 +94,14 @@ CONFIG_SPIRAM_SPEED_40M=y CONFIG_SPIRAM=y CONFIG_SPIRAM_BOOT_INIT=y CONFIG_SPIRAM_USE_MALLOC=y +#CONFIG_SPIRAM_USE_MEMMAP=y => Does not work: "cam_dma_config(306): frame buffer malloc failed" CONFIG_SPIRAM_MEMTEST=y CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384 CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=40960 CONFIG_SPIRAM_CACHE_WORKAROUND=y CONFIG_SPIRAM_IGNORE_NOTFOUND=y -CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y -CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y +#CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y => Leads to memory fragmentation, see https://github.com/jomjol/AI-on-the-edge-device/issues/2200 +#CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y => Leads to memory fragmentation, see https://github.com/jomjol/AI-on-the-edge-device/issues/2200 CONFIG_ESP_INT_WDT_TIMEOUT_MS=300 @@ -122,7 +123,7 @@ CONFIG_MQTT_TASK_CORE_SELECTION_ENABLED=y CONFIG_MQTT_USE_CORE_0=y CONFIG_MQTT_USE_CUSTOM_CONFIG=y #CONFIG_MQTT_OUTBOX_EXPIRED_TIMEOUT_MS=5000 -CONFIG_MQTT_CUSTOM_OUTBOX=y +#CONFIG_MQTT_CUSTOM_OUTBOX=y # -> Use custom outbox in components/jomjol_mqtt/mqtt_outbox.h/cpp. If USE_PSRAM is enabled in there, it will save 10 kBytes of internal RAM. How ever it also leads to memory fragmentation, see https://github.com/jomjol/AI-on-the-edge-device/issues/2200 CONFIG_FREERTOS_TASK_FUNCTION_WRAPPER=n diff --git a/sd-card/html/graph.html b/sd-card/html/graph.html index 315186b3..8dcad980 100644 --- a/sd-card/html/graph.html +++ b/sd-card/html/graph.html @@ -50,39 +50,19 @@ /* Catch empty fields */ if (value == "" || isNaN(value)) { - if (traceValue.y.length > 0) { - value = traceValue.y[traceValue.y.length-1]; - } - else { - value = 0; - } + value = NaN; } if (preValue == "" || isNaN(preValue)) { - if (tracePreValue.y.length > 0) { - preValue = tracePreValue.y[tracePreValue.y.length-1]; - } - else { - preValue = 0; - } + preValue = NaN; } if (changeRate == "" || isNaN(changeRate)) { - if (traceChangeRate.y.length > 0) { - changeRate = traceChangeRate.y[traceChangeRate.y.length-1]; - } - else { - changeRate = 0; - } + changeRate = NaN; } if (changeAbsolute == "" || isNaN(changeAbsolute)) { - if (traceChangeAbsolute.y.length > 0) { - changeAbsolute = traceChangeAbsolute.y[traceChangeAbsolute.y.length-1]; - } - else { - changeAbsolute = 0; - } + changeAbsolute = NaN; } traceValue.y.push(value); @@ -104,6 +84,8 @@ var offsetValue = traceValue.y[0]; var offsetPreValue = tracePreValue.y[0]; + traceValue.connectgaps = true; + if (showRrelativeValues) { traceValue.y.forEach(function(part, index, arr) { arr[index] = arr[index] - offsetValue;