From 40a1aa04300e29c047afd19f8552d46353c09c08 Mon Sep 17 00:00:00 2001 From: jomjol <30766535+jomjol@users.noreply.github.com> Date: Sun, 20 Aug 2023 21:49:28 +0200 Subject: [PATCH] Update submodules, include only needed layers of tflite (#2586) * Initial version * Working Version * Update * Update main.cpp * Updated Docu --- Changelog.md | 16 ++++++- code/README.md | 9 ++++ code/components/esp-nn | 2 +- code/components/esp32-camera | 2 +- .../jomjol_fileserver_ota/CMakeLists.txt | 2 +- .../components/jomjol_influxdb/CMakeLists.txt | 2 +- .../jomjol_tfliteclass/CTfLiteClass.cpp | 45 ++++++++++++++++--- .../jomjol_tfliteclass/CTfLiteClass.h | 14 ++++-- code/components/tflite-micro-esp-examples | 2 +- code/dependencies.lock | 8 +++- code/include/defines.h | 2 +- code/main/main.cpp | 1 + 12 files changed, 89 insertions(+), 16 deletions(-) diff --git a/Changelog.md b/Changelog.md index 88e4303d..4a13a533 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,8 +1,22 @@ +## [unreleased] - 2023-08-20 + +### Changes + +For a full list of changes see [Full list of changes](https://github.com/jomjol/AI-on-the-edge-device/compare/rolling...v15.3.0) + +#### Changed + + - Updates submodules (esp-nn, tflite-micro-example, esp-camera) + + - Explicitly included needed tflite network layers (instead of all) , resulting in much smaller firmware size + + + ## [15.3.0] - 2023-07-22 ### Changes -For a full list of changes see [Full list of changes](https://github.com/jomjol/AI-on-the-edge-device/compare/v15.2.1...v15.2.4) +For a full list of changes see [Full list of changes](https://github.com/jomjol/AI-on-the-edge-device/compare/v15.3.0...v15.2.4) #### Changed diff --git a/code/README.md b/code/README.md index 0eb5cac4..d7dce832 100644 --- a/code/README.md +++ b/code/README.md @@ -8,6 +8,15 @@ git checkout rolling git submodule update --init ``` +## Update Submodules +``` +cd /components/submodule-name (e.g. tflite-micro-example) +git checkout VERSION (e.g. HASH of latest tflite-micro-example build) +cd ../../ (auf Ebene von code) +git submodule update --init +``` +Evt. muss man vorher noch einige Verzeichnisse in compenents von Hand löschen, da sie beim checkout nicht gelöscht wurden (vor update -- init) + ## Build and Flash within terminal See further down to build it within an IDE. ### Compile diff --git a/code/components/esp-nn b/code/components/esp-nn index 6b3ef8e2..1a35708d 160000 --- a/code/components/esp-nn +++ b/code/components/esp-nn @@ -1 +1 @@ -Subproject commit 6b3ef8e226a05554a6d874f6456f5ca1771c01c2 +Subproject commit 1a35708d93c47b695f1da6da3ac49f5c253910c4 diff --git a/code/components/esp32-camera b/code/components/esp32-camera index 5c8349f4..c0c17bd3 160000 --- a/code/components/esp32-camera +++ b/code/components/esp32-camera @@ -1 +1 @@ -Subproject commit 5c8349f4cf169c8a61283e0da9b8cff10994d3f3 +Subproject commit c0c17bd3de5ea575b775db6cbef84befd020d524 diff --git a/code/components/jomjol_fileserver_ota/CMakeLists.txt b/code/components/jomjol_fileserver_ota/CMakeLists.txt index 15fe07b1..d530eb41 100644 --- a/code/components/jomjol_fileserver_ota/CMakeLists.txt +++ b/code/components/jomjol_fileserver_ota/CMakeLists.txt @@ -2,6 +2,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*) idf_component_register(SRCS ${app_sources} INCLUDE_DIRS "." "../../include" "miniz" - REQUIRES vfs tflite-lib esp_http_server app_update esp_http_client nvs_flash jomjol_tfliteclass jomjol_flowcontroll spiffs jomjol_helper jomjol_controlGPIO) + REQUIRES vfs esp_http_server app_update esp_http_client nvs_flash jomjol_tfliteclass jomjol_flowcontroll spiffs jomjol_helper jomjol_controlGPIO) diff --git a/code/components/jomjol_influxdb/CMakeLists.txt b/code/components/jomjol_influxdb/CMakeLists.txt index 47330bd5..54513408 100644 --- a/code/components/jomjol_influxdb/CMakeLists.txt +++ b/code/components/jomjol_influxdb/CMakeLists.txt @@ -2,6 +2,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*) idf_component_register(SRCS ${app_sources} INCLUDE_DIRS "." - REQUIRES tflite-lib esp_http_client jomjol_logfile) + REQUIRES esp_http_client jomjol_logfile) diff --git a/code/components/jomjol_tfliteclass/CTfLiteClass.cpp b/code/components/jomjol_tfliteclass/CTfLiteClass.cpp index 1f8d1daf..1cc76bef 100644 --- a/code/components/jomjol_tfliteclass/CTfLiteClass.cpp +++ b/code/components/jomjol_tfliteclass/CTfLiteClass.cpp @@ -12,6 +12,31 @@ static const char *TAG = "TFLITE"; +/// Static Resolver muss mit allen Operatoren geladen Werden, die benöägit werden - ABER nur 1x --> gesonderte Funktion ///////////////////////////// +static bool MakeStaticResolverDone = false; +static tflite::MicroMutableOpResolver<15> resolver; + +void MakeStaticResolver() +{ + if (MakeStaticResolverDone) + return; + + MakeStaticResolverDone = true; + + resolver.AddFullyConnected(); + resolver.AddReshape(); + resolver.AddSoftmax(); + resolver.AddConv2D(); + resolver.AddMaxPool2D(); + resolver.AddQuantize(); + resolver.AddMul(); + resolver.AddAdd(); + resolver.AddLeakyRelu(); + resolver.AddDequantize(); +} +//////////////////////////////////////////////////////////////////////////////////////// + + float CTfLiteClass::GetOutputValue(int nr) { TfLiteTensor* output2 = this->interpreter->output(0); @@ -179,16 +204,20 @@ bool CTfLiteClass::LoadInputImageBasis(CImageBasis *rs) } + bool CTfLiteClass::MakeAllocate() { - static tflite::AllOpsResolver resolver; + + MakeStaticResolver(); + #ifdef DEBUG_DETAIL_ON LogFile.WriteHeapInfo("CTLiteClass::Alloc start"); #endif LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CTfLiteClass::MakeAllocate"); - this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize, this->error_reporter); + this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize); +// this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize, this->error_reporter); if (this->interpreter) { @@ -285,6 +314,7 @@ bool CTfLiteClass::ReadFileToModel(std::string _fn) bool CTfLiteClass::LoadModel(std::string _fn) { #ifdef SUPRESS_TFLITE_ERRORS +// this->error_reporter = new tflite::ErrorReporter; this->error_reporter = new tflite::OwnMicroErrorReporter; #else this->error_reporter = new tflite::MicroErrorReporter; @@ -320,16 +350,21 @@ CTfLiteClass::CTfLiteClass() CTfLiteClass::~CTfLiteClass() { delete this->interpreter; - delete this->error_reporter; +// delete this->error_reporter; psram_free_shared_tensor_arena_and_model_memory(); } - +#ifdef SUPRESS_TFLITE_ERRORS namespace tflite { +//tflite::ErrorReporter +// int OwnMicroErrorReporter::Report(const char* format, va_list args) + int OwnMicroErrorReporter::Report(const char* format, va_list args) { return 0; } -} +} +#endif + diff --git a/code/components/jomjol_tfliteclass/CTfLiteClass.h b/code/components/jomjol_tfliteclass/CTfLiteClass.h index 8de503da..86ae7d05 100644 --- a/code/components/jomjol_tfliteclass/CTfLiteClass.h +++ b/code/components/jomjol_tfliteclass/CTfLiteClass.h @@ -3,8 +3,12 @@ #ifndef CTFLITECLASS_H #define CTFLITECLASS_H -#include "tensorflow/lite/micro/all_ops_resolver.h" -#include "tensorflow/lite/micro/micro_error_reporter.h" +#include "tensorflow/lite/micro/micro_mutable_op_resolver.h" +#include "tensorflow/lite/micro/micro_interpreter.h" +#include "tensorflow/lite/micro/micro_mutable_op_resolver.h" +#include "tensorflow/lite/micro/kernels/micro_ops.h" + +#include "tensorflow/lite/micro/tflite_bridge/micro_error_reporter.h" #include "tensorflow/lite/micro/micro_interpreter.h" #include "tensorflow/lite/schema/schema_generated.h" #include "tensorflow/lite/micro/kernels/micro_ops.h" @@ -13,6 +17,8 @@ #include "CImageBasis.h" + + #ifdef SUPRESS_TFLITE_ERRORS #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/micro/compatibility.h" @@ -27,6 +33,7 @@ namespace tflite { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #endif + class CTfLiteClass { protected: @@ -34,7 +41,6 @@ class CTfLiteClass const tflite::Model* model; tflite::MicroInterpreter* interpreter; TfLiteTensor* output = nullptr; - static tflite::AllOpsResolver resolver; int kTensorArenaSize; uint8_t *tensor_arena; @@ -68,4 +74,6 @@ class CTfLiteClass int ReadInputDimenstion(int _dim); }; +void MakeStaticResolver(); + #endif //CTFLITECLASS_H \ No newline at end of file diff --git a/code/components/tflite-micro-esp-examples b/code/components/tflite-micro-esp-examples index 095f55a6..1ccd7e14 160000 --- a/code/components/tflite-micro-esp-examples +++ b/code/components/tflite-micro-esp-examples @@ -1 +1 @@ -Subproject commit 095f55a6ee0b9d51f59a7a05d405df2e6336c178 +Subproject commit 1ccd7e14ac2ff7540502f1b422eae21bab799355 diff --git a/code/dependencies.lock b/code/dependencies.lock index 6d9f7a29..16628b30 100644 --- a/code/dependencies.lock +++ b/code/dependencies.lock @@ -1,3 +1,9 @@ -manifest_hash: 63f5c6c9f0bcebc7b9ca12d2aa8b26b2c5f5218d377dc4b2375d9b9ca1df7815 +dependencies: + idf: + component_hash: null + source: + type: idf + version: 5.0.2 +manifest_hash: f880feca80f04921fc95fd31e9c2936b9896764c15a62f6e2d312c57a62a36db target: esp32 version: 1.0.0 diff --git a/code/include/defines.h b/code/include/defines.h index a289da66..641a0694 100644 --- a/code/include/defines.h +++ b/code/include/defines.h @@ -173,7 +173,7 @@ fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); \ exit(1); \ } - #define SUPRESS_TFLITE_ERRORS // use, to avoid error messages from TFLITE + // #define SUPRESS_TFLITE_ERRORS // use, to avoid error messages from TFLITE // connect_wlan.cpp diff --git a/code/main/main.cpp b/code/main/main.cpp index 1e12c0ac..a6774eab 100644 --- a/code/main/main.cpp +++ b/code/main/main.cpp @@ -721,6 +721,7 @@ std::vector splitString(const std::string& str) { } + /*bool replace_all(std::string& s, std::string const& toReplace, std::string const& replaceWith) { std::string buf; std::size_t pos = 0;