diff --git a/.gitignore b/.gitignore index fdc632b6..e0ca791d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .code-workspace /sd-card/htm./.vscode/ /code/build +/sd-card/html/debug/ CMakeLists.txt.user CMakeCache.txt diff --git a/Changelog.md b/Changelog.md index 009eb868..fe201224 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,151 @@ # Versions +##### 10.6.2 - Stability Increase (2022-07-24) + +- **NEW 10.6.2**: ignore hidden files in model selection (configuration page) + +- **NEW 10.6.1**: Revoke esp32cam & tflite update + +- **NEW 10.6.1**: Bug Fix: tflite-filename with ".", HTML spelling error + +- IndluxDB: direct injection into InfluxDB - thanks to **[wetneb](https://github.com/wetneb)** + +- MQTT: implemented "Retain Flag" and extend with absolute Change (in addition to rate) + +- `config.ini`: removal of modelsize (readout from tflite) + +- Updated analog neural network file (`ana1000s2.tflite`) & digital neural network file (`dig1400s2q.tflite`) + +- TFMicro/Lite: Update (espressif Version 20220716) + +- Updated esp32cam (v20220716) + +- ESP-IDF: Update to 4.4 + +- Internal update (CNN algorithm optimizations, reparation for new neural network type) + +- Bug Fix: no time with fixed IP, Postprocessing, MQTT + + + +##### 10.5.2 - Stability Increase (2022-02-22) + +- NEW 10.5.2: Bug Fix: wrong `firmware.bin` (no rate update) +- NEW 10.5.1: Bug Fix: wrong return value, rate value & PreValue status, HTML: SSID & IP were not displayed +- MQTT: changed wifi naming to "wifiRSSI" +- HTML: check selectable values for consistency +- Refactoring of check postprocessing consistency (e.g. max rate, negative rate, ...) +- Bug Fix: corrected error in "Check Consistency Increase" + + + +##### 10.4.0 - Stability Increase (2022-02-12) + +- Graphical configuration: select available neural network files (*.tfl, *.tflite) from drop down menu +- OTA-update: add option to upload tfl / tflite files to the correct location (`/config/`) + - In the future the new files will also be copied to the `firmware` directory of the repository +- Added Wifi RSSI to MQTT information +- Updated analog neural network file (`ana-s3-q-20220105.tflite`) +- Updated digital neural network file (`dig-s1-q-20220102.tflite`) +- Updated build environment to `Espressif 3.5.0` + + + +##### 10.3.0 - Stability Increase (2022-01-29) + +- Implemented LED flash dimming (`LEDIntensity`). + Remark: as auto illumination in the camera is used, this is rather for energy saving. It will not help reducing reflections +- Additional camera parameters: saturation, contrast (although not too much impact yet) +- Some readings will have removable "N"s that can not be removed automatically and are handled with an "error" --> no return value in the field "value" anymore (still reported back via field "raw value") +- Updated esp32 camera hardware driver +- Bug fix: MQTT, HTML improvements + +**ATTENTION: The new ESP32 camera hardware driver is much more stable on newer OV2640 versions (no or much less reboots) but seems to be not fully compatible with older versions.** + +* If you have problem with stalled systems you can try the following + - Update the parameter `ImageQuality` to `12` instead of current value `5` (manually in the `config.ini`) + + - If this is not helping, you might need to update your hardware or stay with version 9.2 + +##### 10.2.0 - Stability Increase (2022-01-14) + +- Due to the updated camera driver, the image looks different and a new setup might be needed + + - Update reference image + - Update Alignment marks + +- Reduce reboot due to camera problems + +- Update esp32-camera to new version (master as of 2022-01-09) + + + +##### 10.1.1 - Stability Increase (2022-01-12) + +- Bug Fix MQTT problem +- Issue: + - Changing from v9.x to 10.x the MQTT-parameter "Topic" was renamed into "MainTopic" to address multiple number meters. This renaming should have been done automatically in the background within the graphical configuration, but was not working. Instead the parameter "Topic" was deleted and "MainTopic" was set to disabled and "undefined". +- ToDo + - Update the `html.zip` + - If old `config.ini` available: copy it to `/config`, open the graphical configuration and save it again. + - If old `config.ini` not available: reset the parameter "MainTopic" within the `config.ini` manually + - Reboot + +##### 10.1.0 - Stability Increase (2022-01-09) + +- Reduce ESP32 frequency to 160MHz + +- Update tflite (new source: https://github.com/espressif/tflite-micro-esp-examples) + +- Update analog neural network (ana-s3-q-20220105.tflite) + +- Update digital neural network (dig-s1-q-20220102.tflite) + +- Increased web-server buffers +- bug fix: compiler compatibility + +##### 10.0.2 - Stability Increase (2022-01-01) + +- NEW v10.0.2: Corrected JSON error + +- Updated compiler toolchain to ESP-IDF 4.3 + +- Removal of memory leak + +- Improved error handling during startup (check PSRAM and camera with remark in logfile) + +- MQTT: implemented raw value additionally, removal of regex contrain + +- Normalized Parameter ``MaxRateValue`` to "change per minute" + +- HTML: improved input handling + +- Corrected error handling: in case of error the old value, rate, timestamp are not transmitted any more + + + +##### 9.2.0 - External Illumination (2021-12-02) + +- Direct JSON access: ``http://IP-ADRESS/json`` +- Error message in log file in case camera error during startup +- Upgrade analog CNN to v9.1.0 +- Upgrade digital CNN to v13.3.0 (added new images) +- html: support of different ports + +##### 9.1.1 - External Illumination (2021-11-16) + +- NEW 9.1.1 bug fix: LED implemenetation +- External LEDs: change control mode (resolve bug with more than 2 LEDs) +- Additional info into log file +- Bug fix: decimal shift, html, log file + +##### 9.0.0 - External Illumination (2021-10-23) + +* Implementation of external illumination to adjust positioning, brightness and color of the illumination now set individually + * Technical details can be found in the wiki: https://github.com/jomjol/AI-on-the-edge-device/wiki/External-LED + +* New housing published for external LEDs and small clearing: https://www.thingiverse.com/thing:5028229 + ##### 8.5.0 - Multi Meter Support (2021-10-07) diff --git a/README.md b/README.md index 0ad9125c..bd6e2fdc 100644 --- a/README.md +++ b/README.md @@ -33,173 +33,35 @@ If you have any technical topics, you can file a issue in this repository. In other cases you can contact the developer via email: ------- -## Coming next - -* Automated update of the neural network file (tflite) to make the learing of additional pictures much easier and automated (GitHub action) -* New "hyprid" neural network for digital numbers --> allowing the detection of intermediate states ("ring between two numbers") as a subdigit - ------ ## Change log -### Known Issues - -* Slow response of web server during picture analysis - **General remark:** Besides the file `firmware.bin`, typically the content of `/html` will need to be updated! ------ -##### 10.6.2 +##### 11.0.0 - Intermediate Digits -- ignore hidden files in model selection (configuration page) +- Implementation of new CNN types to detect intermediate values of digits with rolling numbers + - By default the old algo (0, 1, ..., 9, "N") is active (due to the limited types of digits trained so far) + - Activation can be done by selection a tflite file with the new trained model in the 'config.ini' + - **Details can be found in the [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Neural-Network-Types)** (different types, trained image types, naming convention) -##### 10.6.1 - Stability Increase (2022-07-24) +- Updated neural network files (and adaption to new naming convention) -- **NEW 10.6.1**: Revoke esp32cam & tflite update +- Published a tool to download and combine log files - **Thanks to ** -- **NEW 10.6.1**: Bug Fix: tflite-filename with ".", HTML spelling error + - Files see ['/tools/logfile-tool'](tbd), How-to see [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Gasmeter-Log-Downloader) -- IndluxDB: direct injection into InfluxDB - thanks to **[wetneb](https://github.com/wetneb)** - -- MQTT: implemented "Retain Flag" and extend with absolute Change (in addition to rate) - -- `config.ini`: removal of modelsize (readout from tflite) - -- Updated analog neural network file (`ana1000s2.tflite`) & digital neural network file (`dig1400s2q.tflite`) - -- TFMicro/Lite: Update (espressif Version 20220716) - -- Updated esp32cam (v20220716) - -- ESP-IDF: Update to 4.4 - -- Internal update (CNN algorithm optimizations, reparation for new neural network type) - -- Bug Fix: no time with fixed IP, Postprocessing, MQTT +- Bug Fix: InfluxDB enabling in grahic configuration -##### 10.5.2 - Stability Increase (2022-02-22) - -- NEW 10.5.2: Bug Fix: wrong `firmware.bin` (no rate update) -- NEW 10.5.1: Bug Fix: wrong return value, rate value & PreValue status, HTML: SSID & IP were not displayed -- MQTT: changed wifi naming to "wifiRSSI" -- HTML: check selectable values for consistency -- Refactoring of check postprocessing consistency (e.g. max rate, negative rate, ...) -- Bug Fix: corrected error in "Check Consistency Increase" - - - -##### 10.4.0 - Stability Increase (2022-02-12) - -- Graphical configuration: select available neural network files (*.tfl, *.tflite) from drop down menu -- OTA-update: add option to upload tfl / tflite files to the correct location (`/config/`) - - In the future the new files will also be copied to the `firmware` directory of the repository -- Added Wifi RSSI to MQTT information -- Updated analog neural network file (`ana-s3-q-20220105.tflite`) -- Updated digital neural network file (`dig-s1-q-20220102.tflite`) -- Updated build environment to `Espressif 3.5.0` - - - -##### 10.3.0 - Stability Increase (2022-01-29) - -- Implemented LED flash dimming (`LEDIntensity`). - Remark: as auto illumination in the camera is used, this is rather for energy saving. It will not help reducing reflections -- Additional camera parameters: saturation, contrast (although not too much impact yet) -- Some readings will have removable "N"s that can not be removed automatically and are handled with an "error" --> no return value in the field "value" anymore (still reported back via field "raw value") -- Updated esp32 camera hardware driver -- Bug fix: MQTT, HTML improvements - -**ATTENTION: The new ESP32 camera hardware driver is much more stable on newer OV2640 versions (no or much less reboots) but seems to be not fully compatible with older versions.** - -* If you have problem with stalled systems you can try the following - - Update the parameter `ImageQuality` to `12` instead of current value `5` (manually in the `config.ini`) - - - If this is not helping, you might need to update your hardware or stay with version 9.2 - -##### 10.2.0 - Stability Increase (2022-01-14) - -- Due to the updated camera driver, the image looks different and a new setup might be needed - - Update reference image - - Update Alignment marks - -- Reduce reboot due to camera problems - -- Update esp32-camera to new version (master as of 2022-01-09) - - - -##### 10.1.1 - Stability Increase (2022-01-12) - -- Bug Fix MQTT problem -- Issue: - - Changing from v9.x to 10.x the MQTT-parameter "Topic" was renamed into "MainTopic" to address multiple number meters. This renaming should have been done automatically in the background within the graphical configuration, but was not working. Instead the parameter "Topic" was deleted and "MainTopic" was set to disabled and "undefined". -- ToDo - - Update the `html.zip` - - If old `config.ini` available: copy it to `/config`, open the graphical configuration and save it again. - - If old `config.ini` not available: reset the parameter "MainTopic" within the `config.ini` manually - - Reboot - -##### 10.1.0 - Stability Increase (2022-01-09) - -- Reduce ESP32 frequency to 160MHz - -- Update tflite (new source: https://github.com/espressif/tflite-micro-esp-examples) - -- Update analog neural network (ana-s3-q-20220105.tflite) - -- Update digital neural network (dig-s1-q-20220102.tflite) - -- Increased web-server buffers -- bug fix: compiler compatibility - -##### 10.0.2 - Stability Increase (2022-01-01) - -- NEW v10.0.2: Corrected JSON error - -- Updated compiler toolchain to ESP-IDF 4.3 - -- Removal of memory leak - -- Improved error handling during startup (check PSRAM and camera with remark in logfile) - -- MQTT: implemented raw value additionally, removal of regex contrain - -- Normalized Parameter ``MaxRateValue`` to "change per minute" - -- HTML: improved input handling - -- Corrected error handling: in case of error the old value, rate, timestamp are not transmitted any more - - - -##### 9.2.0 - External Illumination (2021-12-02) - -- Direct JSON access: ``http://IP-ADRESS/json`` -- Error message in log file in case camera error during startup -- Upgrade analog CNN to v9.1.0 -- Upgrade digital CNN to v13.3.0 (added new images) -- html: support of different ports - -##### 9.1.1 - External Illumination (2021-11-16) - -- NEW 9.1.1 bug fix: LED implemenetation -- External LEDs: change control mode (resolve bug with more than 2 LEDs) -- Additional info into log file -- Bug fix: decimal shift, html, log file - -##### 9.0.0 - External Illumination (2021-10-23) - -* Implementation of external illumination to adjust positioning, brightness and color of the illumination now set individually - * Technical details can be found in the wiki: https://github.com/jomjol/AI-on-the-edge-device/wiki/External-LED - -* New housing published for external LEDs and small clearing: https://www.thingiverse.com/thing:5028229 - - +## Tools +* Logfile downloader and combiner (Thx to [reserve85](https://github.com/reserve85)) + * Files see ['/tools/logfile-tool'](tbd), How-to see [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Gasmeter-Log-Downloader) @@ -213,6 +75,10 @@ There are some ideas and feature requests which are not followed currently - mai ## History +##### 10.6.2 - Stability Increase (2022-07-24) + +##### 9.2.0 - External Illumination (2021-12-02) + ##### 8.5.0 Multi Meter Support (2021-10-07) ##### 7.1.2 MQTT-Update - (2021-06-17) diff --git a/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp b/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp index 2e25b02f..6417f8d1 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp @@ -760,7 +760,7 @@ bool ClassFlowCNNGeneral::doNeuralNetwork(string time) _fit = _val + _valminus; } - if (result >= 10) + if (result > 10) result = result - 10; if (result < 0) result = result + 10; @@ -872,11 +872,14 @@ std::vector ClassFlowCNNGeneral::GetHTMLInfo() for (int _ana = 0; _ana < GENERAL.size(); ++_ana) for (int i = 0; i < GENERAL[_ana]->ROI.size(); ++i) { + printf("Image: %d\n", (int) GENERAL[_ana]->ROI[i]->image); + if (GENERAL[_ana]->ROI[i]->image) + { if (GENERAL[_ana]->name == "default") GENERAL[_ana]->ROI[i]->image->SaveToFile(FormatFileName("/sdcard/img_tmp/" + GENERAL[_ana]->ROI[i]->name + ".bmp")); else GENERAL[_ana]->ROI[i]->image->SaveToFile(FormatFileName("/sdcard/img_tmp/" + GENERAL[_ana]->name + "_" + GENERAL[_ana]->ROI[i]->name + ".bmp")); - + } HTMLInfo *zw = new HTMLInfo; if (GENERAL[_ana]->name == "default") diff --git a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp index 751d25af..0351dc17 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp @@ -586,6 +586,8 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req) { std::vector htmlinfo; htmlinfo = GetAllDigital(); + printf("After getClassFlowControll::GetAllDigital\n"); + for (int i = 0; i < htmlinfo.size(); ++i) { if (_fn == htmlinfo[i]->filename) diff --git a/code/components/jomjol_tfliteclass/server_tflite.cpp b/code/components/jomjol_tfliteclass/server_tflite.cpp index d6683737..28619fcf 100644 --- a/code/components/jomjol_tfliteclass/server_tflite.cpp +++ b/code/components/jomjol_tfliteclass/server_tflite.cpp @@ -314,6 +314,7 @@ esp_err_t handler_wasserzaehler(httpd_req_t *req) std::vector htmlinfodig; htmlinfodig = tfliteflow.GetAllDigital(); + for (int i = 0; i < htmlinfodig.size(); ++i) { if (tfliteflow.GetTypeDigital() == Digital) diff --git a/code/main/version.cpp b/code/main/version.cpp index c5275f20..981409cc 100644 --- a/code/main/version.cpp +++ b/code/main/version.cpp @@ -1,4 +1,4 @@ -const char* GIT_REV="0e7c600"; +const char* GIT_REV="9695dba"; const char* GIT_TAG=""; -const char* GIT_BRANCH="master"; -const char* BUILD_TIME="2022-07-24 18:59"; \ No newline at end of file +const char* GIT_BRANCH="rolling"; +const char* BUILD_TIME="2022-08-13 14:15"; \ No newline at end of file diff --git a/code/main/version.h b/code/main/version.h index ac0d49de..cc20446e 100644 --- a/code/main/version.h +++ b/code/main/version.h @@ -13,7 +13,7 @@ extern "C" #include "Helper.h" #include -const char* GIT_BASE_BRANCH = "master - v10.6.1 - 2022-07-24"; +const char* GIT_BASE_BRANCH = "master - v11.0.0 - 2022-08-13"; const char* git_base_branch(void) diff --git a/code/version.cpp b/code/version.cpp index c5275f20..981409cc 100644 --- a/code/version.cpp +++ b/code/version.cpp @@ -1,4 +1,4 @@ -const char* GIT_REV="0e7c600"; +const char* GIT_REV="9695dba"; const char* GIT_TAG=""; -const char* GIT_BRANCH="master"; -const char* BUILD_TIME="2022-07-24 18:59"; \ No newline at end of file +const char* GIT_BRANCH="rolling"; +const char* BUILD_TIME="2022-08-13 14:15"; \ No newline at end of file diff --git a/firmware/bootloader.bin b/firmware/bootloader.bin index 737615bb..735cc2a3 100644 Binary files a/firmware/bootloader.bin and b/firmware/bootloader.bin differ diff --git a/firmware/firmware.bin b/firmware/firmware.bin index c684bb99..5e1d2447 100644 Binary files a/firmware/firmware.bin and b/firmware/firmware.bin differ diff --git a/firmware/html.zip b/firmware/html.zip index 0652a9d1..a9d2a5cc 100644 Binary files a/firmware/html.zip and b/firmware/html.zip differ diff --git a/sd-card/config/ana_i32s100dr-v1.0-q.tflite b/sd-card/config/ana-class100_i32s100dr-v1.0_q.tflite similarity index 100% rename from sd-card/config/ana_i32s100dr-v1.0-q.tflite rename to sd-card/config/ana-class100_i32s100dr-v1.0_q.tflite diff --git a/sd-card/config/ana-cont_1101_s2_q.tflite b/sd-card/config/ana-cont_1101_s2_q.tflite new file mode 100644 index 00000000..fb5a38cf Binary files /dev/null and b/sd-card/config/ana-cont_1101_s2_q.tflite differ diff --git a/sd-card/config/ana1000s2.tflite b/sd-card/config/ana1000s2.tflite deleted file mode 100644 index 72ae8ba2..00000000 Binary files a/sd-card/config/ana1000s2.tflite and /dev/null differ diff --git a/sd-card/config/config.ini b/sd-card/config/config.ini index 94a9c5cd..290d1213 100644 --- a/sd-card/config/config.ini +++ b/sd-card/config/config.ini @@ -21,7 +21,8 @@ FlipImageSize = false /config/ref1.jpg 442 142 [Digits] -Model = /config/dig1400s2q.tflite +Model = /config/dig-class11_1411_s2_q.tflite +CNNGoodThreshold = 0.5 ;LogImageLocation = /log/digit ;LogfileRetentionInDays = 3 main.dig1 294 126 30 54 @@ -29,7 +30,8 @@ main.dig2 343 126 30 54 main.dig3 391 126 30 54 [Analog] -Model = /config/ana1000s2.tflite +Model = /config/ana-cont_1101_s2_q.tflite +CNNGoodThreshold = 0.5 ;LogImageLocation = /log/analog ;LogfileRetentionInDays = 3 ExtendedResolution = true @@ -56,6 +58,14 @@ CheckDigitIncreaseConsistency = false ;ClientID = wasser ;user = USERNAME ;password = PASSWORD +;SetRetainFlag = true + +;[InfluxDB] +;Uri = undefined +;Database = +;Measurement = undefined +;user = undefined +;password = undefined ;[GPIO] ;MainTopicMQTT = wasserzaehler/GPIO diff --git a/sd-card/config/dig-s1d-md-v1.0-q.tflite b/sd-card/config/dig-class100_s1d-md-v1.0_q.tflite similarity index 100% rename from sd-card/config/dig-s1d-md-v1.0-q.tflite rename to sd-card/config/dig-class100_s1d-md-v1.0_q.tflite diff --git a/sd-card/config/dig-class11_1411_s2_q.tflite b/sd-card/config/dig-class11_1411_s2_q.tflite new file mode 100644 index 00000000..0ea6a067 Binary files /dev/null and b/sd-card/config/dig-class11_1411_s2_q.tflite differ diff --git a/sd-card/config/dig-cont_0560_s3_q.tflite b/sd-card/config/dig-cont_0560_s3_q.tflite new file mode 100644 index 00000000..f69648ca Binary files /dev/null and b/sd-card/config/dig-cont_0560_s3_q.tflite differ diff --git a/sd-card/config/dig1400s2q.tflite b/sd-card/config/dig1400s2q.tflite deleted file mode 100644 index 99a28f53..00000000 Binary files a/sd-card/config/dig1400s2q.tflite and /dev/null differ diff --git a/sd-card/html/edit_config_param.html b/sd-card/html/edit_config_param.html index 79d86160..b6c7a34a 100644 --- a/sd-card/html/edit_config_param.html +++ b/sd-card/html/edit_config_param.html @@ -1866,6 +1866,7 @@ function UpdateInput() { document.getElementById("Category_Digits_enabled").checked = category["Digits"]["enabled"]; document.getElementById("Category_MQTT_enabled").checked = category["MQTT"]["enabled"]; document.getElementById("Category_GPIO_enabled").checked = category["GPIO"]["enabled"]; + document.getElementById("Category_InfluxDB_enabled").checked = category["InfluxDB"]["enabled"]; setVisible("GPIO_item", category["GPIO"]["enabled"]); WriteParameter(param, category, "MakeImage", "LogImageLocation", true); @@ -2058,6 +2059,9 @@ function UpdateAfterCategoryCheck() { category["Analog"]["enabled"] = document.getElementById("Category_Analog_enabled").checked; category["Digits"]["enabled"] = document.getElementById("Category_Digits_enabled").checked; category["MQTT"]["enabled"] = document.getElementById("Category_MQTT_enabled").checked; + category["InfluxDB"]["enabled"] = document.getElementById("Category_InfluxDB_enabled").checked; + category["GPIO"]["enabled"] = document.getElementById("Category_GPIO_enabled").checked; + UpdateInput(); UpdateInputIndividual(); } diff --git a/sd-card/html/version.txt b/sd-card/html/version.txt index 3b85ad66..1cbb0aa6 100644 --- a/sd-card/html/version.txt +++ b/sd-card/html/version.txt @@ -1 +1 @@ -14.2.2 \ No newline at end of file +14.3.0 \ No newline at end of file diff --git a/tools/logfile-tool/Gasmeter_Logdownloader.dpr b/tools/logfile-tool/Gasmeter_Logdownloader.dpr new file mode 100644 index 00000000..c6b60d53 --- /dev/null +++ b/tools/logfile-tool/Gasmeter_Logdownloader.dpr @@ -0,0 +1,14 @@ +program Gasmeter_Logdownloader; + +uses + Vcl.Forms, + uMain in 'uMain.pas' {Form1}; + +{$R *.res} + +begin + Application.Initialize; + Application.MainFormOnTaskbar := True; + Application.CreateForm(TForm1, Form1); + Application.Run; +end. diff --git a/tools/logfile-tool/Gasmeter_Logdownloader.dproj b/tools/logfile-tool/Gasmeter_Logdownloader.dproj new file mode 100644 index 00000000..670d668d --- /dev/null +++ b/tools/logfile-tool/Gasmeter_Logdownloader.dproj @@ -0,0 +1,546 @@ + + + True + Application + Debug + VCL + Gasmeter_Logdownloader.dpr + Win32 + {EF2C2455-2FD5-4992-8408-A473425308BD} + 18.7 + 1 + + + true + + + true + Base + true + + + true + Base + true + + + true + Cfg_1 + true + true + + + true + Base + true + + + true + Cfg_2 + true + true + + + Gasmeter_Logdownloader + .\$(Platform)\$(Config) + .\$(Platform)\$(Config) + System;Xml;Data;Datasnap;Web;Soap;Vcl;Vcl.Imaging;Vcl.Touch;Vcl.Samples;Vcl.Shell;$(DCC_Namespace) + $(BDS)\bin\delphi_PROJECTICON.ico + $(BDS)\bin\Artwork\Windows\UWP\delphi_UwpDefault_150.png + $(BDS)\bin\Artwork\Windows\UWP\delphi_UwpDefault_44.png + + + Debug + Winapi;System.Win;Data.Win;Datasnap.Win;Web.Win;Soap.Win;Xml.Win;Bde;$(DCC_Namespace) + DBXSqliteDriver;RESTComponents;WclBluetoothFrameworkR;DBXInterBaseDriver;vclactnband;vclFireDAC;dacfmx260;RSVML;tethering;svnui;FireDACADSDriver;vcltouch;vcldb;bindcompfmx;svn;IcsFmxD103Run;inetdb;IcsVclD103Run;FmxTeeUI;fmx;FireDACIBDriver;fmxdae;frxTee26;RSCommon;dacvcl260;fs26;IndyCore;dbexpress;vclx;frxIntIO26;dsnap;FireDACCommon;RSVclCommon;RESTBackendComponents;VCLRESTComponents;TMSVCLUIPackPkgWizDXE12;soapserver;fsTee26;TMSVCLUIPackPkgDXE12;vclie;bindengine;DBXMySQLDriver;CloudService;FireDACMySQLDriver;frx26;FireDACCommonODBC;FireDACCommonDriver;frxIntIOIndy26;inet;S0586_TRVclComponents;bindcompdbx;IndyIPCommon;vcl;IndyIPServer;frxDB26;IndySystem;fsADO26;frxDBX26;dsnapcon;sdac260;FireDACMSAccDriver;fsDB26;fmxFireDAC;vclimg;S0606_BMVclComponents;S0628_EWVclComponents;FireDAC;TeeDB;dOPCP;frxe26;FireDACSqliteDriver;FireDACPgDriver;tdstream260;sdacvcl260;crcontrols260;TMSVCLUIPackPkgXlsDXE12;FMXTee;soaprtl;DbxCommonDriver;Tee;xmlrtl;soapmidas;dac260;vclwinx;fmxobj;rtl;frxADO26;DbxClientDriver;CustomIPTransport;vcldsnap;SynEditDR;bindcomp;appanalytics;TMSVCLUIPackPkgExDXE12;RSFMXSVG;IndyIPClient;IcsCommonD103Run;sdacfmx260;bindcompvcl;TeeUI;dbxcds;VclSmp;adortl;sbridge260;dsnapxml;dbrtl;IndyProtocols;inetdbxpress;RSVclSVG;fmxase;$(DCC_UsePackage) + $(BDS)\bin\default_app.manifest + true + CompanyName=;FileDescription=$(MSBuildProjectName);FileVersion=1.0.0.0;InternalName=;LegalCopyright=;LegalTrademarks=;OriginalFilename=;ProgramID=com.embarcadero.$(MSBuildProjectName);ProductName=$(MSBuildProjectName);ProductVersion=1.0.0.0;Comments= + 1033 + + + true + true + DEBUG;$(DCC_Define) + true + false + true + + + PerMonitorV2 + true + false + + + 0 + RELEASE;$(DCC_Define) + false + 0 + + + PerMonitorV2 + true + + + + MainSource + + +
Form1
+ dfm + + + Cfg_2 + Base + + + Base + + + Cfg_1 + Base + + + + Delphi.Personality.12 + Application + + + + Gasmeter_Logdownloader.dpr + + + + + + Gasmeter_Logdownloader.exe + true + + + + + 1 + + + Contents\MacOS + 1 + + + Contents\MacOS + 0 + + + + + classes + 1 + + + + + library\lib\armeabi-v7a + 1 + + + + + library\lib\armeabi + 1 + + + + + library\lib\mips + 1 + + + + + library\lib\armeabi-v7a + 1 + + + + + res\drawable + 1 + + + + + res\values + 1 + + + + + res\drawable + 1 + + + + + res\drawable-xxhdpi + 1 + + + + + res\drawable-ldpi + 1 + + + + + res\drawable-mdpi + 1 + + + + + res\drawable-hdpi + 1 + + + + + res\drawable-xhdpi + 1 + + + + + res\drawable-small + 1 + + + + + res\drawable-normal + 1 + + + + + res\drawable-large + 1 + + + + + res\drawable-xlarge + 1 + + + + + 1 + + + Contents\MacOS + 1 + + + 0 + + + + + Contents\MacOS + 1 + .framework + + + 0 + + + + + 1 + .dylib + + + 1 + .dylib + + + 1 + .dylib + + + Contents\MacOS + 1 + .dylib + + + 0 + .dll;.bpl + + + + + 1 + .dylib + + + 1 + .dylib + + + 1 + .dylib + + + Contents\MacOS + 1 + .dylib + + + 0 + .bpl + + + + + 0 + + + 0 + + + 0 + + + 0 + + + Contents\Resources\StartUp\ + 0 + + + 0 + + + + + 1 + + + 1 + + + 1 + + + + + 1 + + + 1 + + + 1 + + + + + 1 + + + 1 + + + 1 + + + + + 1 + + + 1 + + + 1 + + + + + 1 + + + 1 + + + 1 + + + + + 1 + + + 1 + + + 1 + + + + + 1 + + + 1 + + + 1 + + + + + 1 + + + + + ..\$(PROJECTNAME).app.dSYM\Contents\Resources\DWARF + 1 + + + ..\$(PROJECTNAME).app.dSYM\Contents\Resources\DWARF + 1 + + + + + 1 + + + 1 + + + + + ..\ + 1 + + + ..\ + 1 + + + + + 1 + + + 1 + + + 1 + + + + + 1 + + + 1 + + + 1 + + + + + ..\ + 1 + + + + + Contents + 1 + + + + + Contents\Resources + 1 + + + + + library\lib\armeabi-v7a + 1 + + + 1 + + + 1 + + + 1 + + + 1 + + + Contents\MacOS + 1 + + + 0 + + + + + 1 + + + 1 + + + + + Assets + 1 + + + Assets + 1 + + + + + Assets + 1 + + + Assets + 1 + + + + + + + + + + + + + True + False + + + 12 + + + + + diff --git a/tools/logfile-tool/Gasmeter_Logdownloader.exe b/tools/logfile-tool/Gasmeter_Logdownloader.exe new file mode 100644 index 00000000..408786ab Binary files /dev/null and b/tools/logfile-tool/Gasmeter_Logdownloader.exe differ diff --git a/tools/logfile-tool/readme.html b/tools/logfile-tool/readme.html new file mode 100644 index 00000000..f22ba24f --- /dev/null +++ b/tools/logfile-tool/readme.html @@ -0,0 +1,14 @@ +

Gasmeter Log-Downloader

+

This small tool downloads the logfiles from your ESP32 and stores the last value of the day in an *.csv file.

+

To use this tool you need to activate the debug logfile in your configuration (Configuration / Debug / Logfile). I go with 30 days of retention in days.

+

It downloads only the past logfiles (yesterday and older).

+

You can define the max. number of Logfiles to download (beginning from newest [yesterday]).

+

I wrote this tool to get a chart of the daily gas consumption to optimize my gas powered heating.

+

Variables to define by yourself:

+
    +
  • URL to Logfile-Path on Device: "http://ESP32-IP-Address/fileserver/log/message/"
    • +
  • Download Logfiles to: enter a valid directory, e.g. "D:\Gaszaehler\Auswertung\Log-Downloads\"
    • +
  • Output CSV-File: enter a valid directory, e.g. "D:\Gaszaehler\Auswertung\DailyValues.csv"
    • +
  • Download Logfiles from past # days: enter the max. number of logfiles you want to download (<= your logfile retention value in your device configuration)
    • +
+

Feel free to optimize and modify it.

\ No newline at end of file diff --git a/tools/logfile-tool/uMain.dfm b/tools/logfile-tool/uMain.dfm new file mode 100644 index 00000000..09d47a6d --- /dev/null +++ b/tools/logfile-tool/uMain.dfm @@ -0,0 +1,111 @@ +object Form1: TForm1 + Left = 0 + Top = 0 + Caption = 'Gasmeter Log-Downloader' + ClientHeight = 521 + ClientWidth = 513 + Color = clBtnFace + Font.Charset = DEFAULT_CHARSET + Font.Color = clWindowText + Font.Height = -11 + Font.Name = 'Tahoma' + Font.Style = [] + OldCreateOrder = False + OnCreate = FormCreate + OnDestroy = FormDestroy + PixelsPerInch = 96 + TextHeight = 13 + object lblImpressum: TLabel + Left = 376 + Top = 468 + Width = 68 + Height = 13 + Caption = 'reserve, 2022' + end + object lbledtURL: TLabeledEdit + Left = 28 + Top = 36 + Width = 273 + Height = 21 + EditLabel.Width = 146 + EditLabel.Height = 13 + EditLabel.Caption = 'URL to Logfile-Path on Device:' + TabOrder = 0 + Text = 'http://192.168.10.65/fileserver/log/message/' + end + object btnDownloadLogfiles: TButton + Left = 28 + Top = 174 + Width = 273 + Height = 25 + Caption = 'Download Logfiles and generate CSV' + TabOrder = 1 + OnClick = btnDownloadLogfilesClick + end + object lbledtMaxLogfilesOnServer: TLabeledEdit + Left = 323 + Top = 36 + Width = 121 + Height = 21 + EditLabel.Width = 173 + EditLabel.Height = 13 + EditLabel.Caption = 'Download logfiles from past # days:' + TabOrder = 2 + Text = '30' + end + object lbledtTargetDirectory: TLabeledEdit + Left = 28 + Top = 84 + Width = 273 + Height = 21 + EditLabel.Width = 103 + EditLabel.Height = 13 + EditLabel.Caption = 'Download Logfiles to:' + TabOrder = 3 + Text = 'C:\Temp\Gas\' + end + object lbledtCsvFile: TLabeledEdit + Left = 28 + Top = 131 + Width = 273 + Height = 21 + EditLabel.Width = 80 + EditLabel.Height = 13 + EditLabel.Caption = 'Output CSV-File:' + TabOrder = 4 + Text = 'C:\Temp\Gas\Values.csv' + end + object redtLog: TRichEdit + Left = 28 + Top = 220 + Width = 273 + Height = 277 + Font.Charset = ANSI_CHARSET + Font.Color = clWindowText + Font.Height = -11 + Font.Name = 'Tahoma' + Font.Style = [] + Lines.Strings = ( + 'redtLog') + ParentFont = False + TabOrder = 5 + Zoom = 100 + end + object idhtp1: TIdHTTP + AllowCookies = True + ProxyParams.BasicAuthentication = False + ProxyParams.ProxyPort = 0 + Request.ContentLength = -1 + Request.ContentRangeEnd = -1 + Request.ContentRangeStart = -1 + Request.ContentRangeInstanceLength = -1 + Request.Accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + Request.BasicAuthentication = False + Request.UserAgent = 'Mozilla/3.0 (compatible; Indy Library)' + Request.Ranges.Units = 'bytes' + Request.Ranges = <> + HTTPOptions = [hoForceEncodeParams] + Left = 572 + Top = 464 + end +end diff --git a/tools/logfile-tool/uMain.pas b/tools/logfile-tool/uMain.pas new file mode 100644 index 00000000..1d13470c --- /dev/null +++ b/tools/logfile-tool/uMain.pas @@ -0,0 +1,174 @@ +unit uMain; + +interface + +uses + Winapi.Windows, Winapi.Messages, System.SysUtils, System.Variants, System.Classes, Vcl.Graphics, System.IOUtils, System.IniFiles, + Vcl.Controls, Vcl.Forms, Vcl.Dialogs, Vcl.StdCtrls, Vcl.ExtCtrls, IdStream, IdBaseComponent, IdComponent, IdTCPConnection, IdTCPClient, + IdHTTP, System.DateUtils, Generics.Collections, Generics.Defaults, Vcl.ComCtrls; + +type + TForm1 = class(TForm) + lbledtURL: TLabeledEdit; + btnDownloadLogfiles: TButton; + lbledtMaxLogfilesOnServer: TLabeledEdit; + lbledtTargetDirectory: TLabeledEdit; + idhtp1: TIdHTTP; + lbledtCsvFile: TLabeledEdit; + redtLog: TRichEdit; + lblImpressum: TLabel; + procedure btnDownloadLogfilesClick(Sender: TObject); + procedure FormCreate(Sender: TObject); + procedure FormDestroy(Sender: TObject); + private + { Private declarations } + mINI: TINIFile; + function DownloadFile(pURL: string; pDestFileName: string): boolean; + function LoadValue(const pFileName: string): Extended; + procedure LoadCSV(const pFileName: string); + procedure SaveCSV(const pFileName: string); + procedure LoadINI; + procedure WriteINI; + public + { Public declarations } + end; + +var + Form1: TForm1; + +implementation + +{$R *.dfm} + +procedure TForm1.btnDownloadLogfilesClick(Sender: TObject); +var + lclDateString: string; + lclFilename: string; + i: Integer; + lclValue: Extended; +begin + redtLog.Clear; + LoadCSV(lbledtCsvFile.Text); + for i := StrToInt(lbledtMaxLogfilesOnServer.Text) downto 1 do + begin + DateTimeToString(lclDateString, 'yyyy-mm-dd', incDay(Now, -i)); + lclFilename := 'log_' + lclDateString + '.txt'; + if (redtLog.FindText(lclDateString, 0, Length(redtLog.Lines.Text), [stWholeWord]) = -1) then + begin + if DownloadFile(lbledtURL.Text + lclFilename, lbledtTargetDirectory.Text + lclFilename) then + begin + lclValue := LoadValue(lbledtTargetDirectory.Text + lclFilename); + redtLog.Lines.Add(lclDateString + ';' + FloatToStrF(lclValue, ffFixed, 8, 2)); + end; + end; + end; + SaveCSV(lbledtCsvFile.Text); +end; + +procedure TForm1.LoadCSV(const pFileName: string); +var + Txt: TextFile; + s: string; +begin + if FileExists(pFileName) then + begin + AssignFile(Txt, pFileName); + Reset(Txt); + while not Eof(Txt) do + begin + Readln(Txt, s); + redtLog.Lines.Add(s); + end; + CloseFile(Txt); + end + else + begin + redtLog.Lines.Add('Date;Value'); + end; +end; + +procedure TForm1.LoadINI; +begin + lbledtURL.Text := mINI.ReadString('MAIN', 'URL', 'http://192.168.10.65/fileserver/log/message/'); + lbledtMaxLogfilesOnServer.Text := mINI.ReadString('MAIN', 'CountLogfiles', '30'); + lbledtTargetDirectory.Text := mINI.ReadString('MAIN', 'Log', 'C:\Temp\Gas\'); + lbledtCsvFile.Text := mINI.ReadString('MAIN', 'CsvFile', 'C:\Temp\Gas\Values.csv'); +end; + +function TForm1.LoadValue(const pFileName: string): Extended; +var + Txt: TextFile; + s: string; + lclStartPos: Integer; + lclEndPos: Integer; +begin + Result := 0; + AssignFile(Txt, pFileName); + Reset(Txt); + while not Eof(Txt) do + begin + Readln(Txt, s); + if (AnsiPos('Value: ', s) <> 0) and (AnsiPos(' Error: no error', s) <> 0) then + begin + lclStartPos := AnsiPos('Value: ', s) + 7; + lclEndPos := AnsiPos(' Error: no error', s) - lclStartPos; + s := StringReplace(s, '.', ',', [rfReplaceAll, rfIgnoreCase]); + Result := StrToFloat(Copy(s, lclStartPos, lclEndPos)); + end; + end; + CloseFile(Txt); +end; + +procedure TForm1.SaveCSV(const pFileName: string); +begin + TFile.WriteAllText(pFileName, redtLog.Lines.Text); +end; + +procedure TForm1.WriteINI; +begin + mINI.WriteString('MAIN', 'URL', lbledtURL.Text); + mINI.WriteString('MAIN', 'CountLogfiles', lbledtMaxLogfilesOnServer.Text); + mINI.WriteString('MAIN', 'Log', lbledtTargetDirectory.Text); + mINI.WriteString('MAIN', 'CsvFile', lbledtCsvFile.Text); +end; + +function TForm1.DownloadFile(pURL: string; pDestFileName: string): boolean; +var + Http: TIdHTTP; + FS: TFileStream; +begin + Result := true; + ForceDirectories(ExtractFileDir(pDestFileName)); + FS := TFileStream.Create(pDestFileName, fmCreate); + try + try + Http := TIdHTTP.Create(nil); + try + Http.Get(pURL, FS); + finally + Http.Free; + end; + finally + FS.Free; + end; + except + DeleteFile(pDestFileName); + Exit(false); + end; +end; + +procedure TForm1.FormCreate(Sender: TObject); +begin + mINI := TINIFile.Create(ExtractFilePath(ParamStr(0)) + 'config.ini'); + LoadINI; + redtLog.Clear; + lblImpressum.Caption := 'reserve, 2022' + #13#10 + 'free to copy and modify' +end; + +procedure TForm1.FormDestroy(Sender: TObject); +begin + WriteINI; + FreeAndNil(mINI); +end; + +end.