Update submodules, include only needed layers of tflite (#2586)

* Initial version

* Working Version

* Update

* Update main.cpp

* Updated Docu

Changelog.md

@@ -1,8 +1,22 @@
+## [unreleased] - 2023-08-20
+
+### Changes
+
+For a full list of changes see [Full list of changes](https://github.com/jomjol/AI-on-the-edge-device/compare/rolling...v15.3.0)
+
+#### Changed
+
+ - Updates submodules (esp-nn, tflite-micro-example, esp-camera)
+
+ - Explicitly included needed tflite network layers (instead of all) , resulting in much smaller firmware size
+
+   
+
 ## [15.3.0] - 2023-07-22
 
 ### Changes
 
-For a full list of changes see [Full list of changes](https://github.com/jomjol/AI-on-the-edge-device/compare/v15.2.1...v15.2.4)
+For a full list of changes see [Full list of changes](https://github.com/jomjol/AI-on-the-edge-device/compare/v15.3.0...v15.2.4)
 
 #### Changed
 

code/README.md

@@ -8,6 +8,15 @@ git checkout rolling
 git submodule update --init
 ```
 
+## Update Submodules
+```
+cd /components/submodule-name (e.g. tflite-micro-example)
+git checkout VERSION (e.g. HASH of latest tflite-micro-example build)
+cd ../../ (auf Ebene von code)
+git submodule update --init
+```
+Evt. muss man vorher noch einige Verzeichnisse in compenents von Hand löschen, da sie beim checkout nicht gelöscht wurden (vor update -- init)
+
 ## Build and Flash within terminal
 See further down to build it within an IDE.
 ### Compile

code/components/jomjol_fileserver_ota/CMakeLists.txt

@@ -2,6 +2,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*)
 
 idf_component_register(SRCS ${app_sources}
                     INCLUDE_DIRS "." "../../include" "miniz"
-                    REQUIRES vfs tflite-lib esp_http_server app_update esp_http_client nvs_flash jomjol_tfliteclass jomjol_flowcontroll spiffs jomjol_helper jomjol_controlGPIO)
+                    REQUIRES vfs esp_http_server app_update esp_http_client nvs_flash jomjol_tfliteclass jomjol_flowcontroll spiffs jomjol_helper jomjol_controlGPIO)
 
 

code/components/jomjol_influxdb/CMakeLists.txt

@@ -2,6 +2,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*)
 
 idf_component_register(SRCS ${app_sources}
                     INCLUDE_DIRS "."
-                    REQUIRES tflite-lib esp_http_client jomjol_logfile)
+                    REQUIRES esp_http_client jomjol_logfile)
 
 

code/components/jomjol_tfliteclass/CTfLiteClass.cpp

@@ -12,6 +12,31 @@
 
 static const char *TAG = "TFLITE";
 
+/// Static Resolver muss mit allen Operatoren geladen Werden, die benöägit werden - ABER nur 1x --> gesonderte Funktion /////////////////////////////
+static bool MakeStaticResolverDone = false;
+static tflite::MicroMutableOpResolver<15> resolver;
+
+void MakeStaticResolver()
+{
+  if (MakeStaticResolverDone)
+    return;
+
+  MakeStaticResolverDone = true;
+
+  resolver.AddFullyConnected();
+  resolver.AddReshape();
+  resolver.AddSoftmax();
+  resolver.AddConv2D();
+  resolver.AddMaxPool2D();
+  resolver.AddQuantize();
+  resolver.AddMul();
+  resolver.AddAdd();
+  resolver.AddLeakyRelu();
+  resolver.AddDequantize();
+}
+////////////////////////////////////////////////////////////////////////////////////////
+
+
 float CTfLiteClass::GetOutputValue(int nr)
 {
     TfLiteTensor* output2 = this->interpreter->output(0);
@@ -179,16 +204,20 @@ bool CTfLiteClass::LoadInputImageBasis(CImageBasis *rs)
 }
 
 
+
 bool CTfLiteClass::MakeAllocate()
 {
-    static tflite::AllOpsResolver resolver;
+
+  MakeStaticResolver();
+
 
     #ifdef DEBUG_DETAIL_ON 
         LogFile.WriteHeapInfo("CTLiteClass::Alloc start");
     #endif
 
     LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CTfLiteClass::MakeAllocate");
-    this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize, this->error_reporter);
+    this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize);
+//    this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize, this->error_reporter);
 
     if (this->interpreter) 
     {
@@ -285,6 +314,7 @@ bool CTfLiteClass::ReadFileToModel(std::string _fn)
 bool CTfLiteClass::LoadModel(std::string _fn)
 {
 #ifdef SUPRESS_TFLITE_ERRORS
+//    this->error_reporter = new tflite::ErrorReporter;
     this->error_reporter = new tflite::OwnMicroErrorReporter;
 #else
     this->error_reporter = new tflite::MicroErrorReporter;
@@ -320,16 +350,21 @@ CTfLiteClass::CTfLiteClass()
 CTfLiteClass::~CTfLiteClass()
 {
   delete this->interpreter;
-  delete this->error_reporter;
+//  delete this->error_reporter;
 
   psram_free_shared_tensor_arena_and_model_memory();
 }        
 
-
+#ifdef SUPRESS_TFLITE_ERRORS
 namespace tflite 
 {
+//tflite::ErrorReporter
+//  int OwnMicroErrorReporter::Report(const char* format, va_list args) 
+
   int OwnMicroErrorReporter::Report(const char* format, va_list args) 
   {
     return 0;
   }
-}  
+} 
+#endif
+ 

code/components/jomjol_tfliteclass/CTfLiteClass.h

@@ -3,8 +3,12 @@
 #ifndef CTFLITECLASS_H
 #define CTFLITECLASS_H
 
-#include "tensorflow/lite/micro/all_ops_resolver.h"
+#include "tensorflow/lite/micro/micro_mutable_op_resolver.h"
-#include "tensorflow/lite/micro/micro_error_reporter.h"
+#include "tensorflow/lite/micro/micro_interpreter.h"
+#include "tensorflow/lite/micro/micro_mutable_op_resolver.h"
+#include "tensorflow/lite/micro/kernels/micro_ops.h"
+
+#include "tensorflow/lite/micro/tflite_bridge/micro_error_reporter.h"
 #include "tensorflow/lite/micro/micro_interpreter.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 #include "tensorflow/lite/micro/kernels/micro_ops.h"
@@ -13,6 +17,8 @@
 
 #include "CImageBasis.h"
 
+
+
 #ifdef SUPRESS_TFLITE_ERRORS
 #include "tensorflow/lite/core/api/error_reporter.h"
 #include "tensorflow/lite/micro/compatibility.h"
@@ -27,6 +33,7 @@ namespace tflite {
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 #endif
 
+
 class CTfLiteClass
 {
     protected:
@@ -34,7 +41,6 @@ class CTfLiteClass
         const tflite::Model* model;
         tflite::MicroInterpreter* interpreter;
         TfLiteTensor* output = nullptr;     
-        static tflite::AllOpsResolver resolver;
 
         int kTensorArenaSize;
         uint8_t *tensor_arena;
@@ -68,4 +74,6 @@ class CTfLiteClass
         int ReadInputDimenstion(int _dim);
 };
 
+void MakeStaticResolver();
+
 #endif //CTFLITECLASS_H

code/dependencies.lock

@@ -1,3 +1,9 @@
-manifest_hash: 63f5c6c9f0bcebc7b9ca12d2aa8b26b2c5f5218d377dc4b2375d9b9ca1df7815
+dependencies:
+  idf:
+    component_hash: null
+    source:
+      type: idf
+    version: 5.0.2
+manifest_hash: f880feca80f04921fc95fd31e9c2936b9896764c15a62f6e2d312c57a62a36db
 target: esp32
 version: 1.0.0

code/include/defines.h

@@ -173,7 +173,7 @@
             fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); \
             exit(1);                                                 \
         }
-    #define SUPRESS_TFLITE_ERRORS // use, to avoid error messages from TFLITE
+    // #define SUPRESS_TFLITE_ERRORS // use, to avoid error messages from TFLITE
 
 
     // connect_wlan.cpp

code/main/main.cpp

@@ -721,6 +721,7 @@ std::vector<std::string> splitString(const std::string& str) {
 }
 
 
+
 /*bool replace_all(std::string& s, std::string const& toReplace, std::string const& replaceWith) {
     std::string buf;
     std::size_t pos = 0;