From 12c748534e4ef41dec1c3270117a99cf56c5deff Mon Sep 17 00:00:00 2001 From: Frank Haverland Date: Sun, 25 Sep 2022 18:36:15 +0200 Subject: [PATCH 1/9] cleanup initial-setup is now part of the release --- .github/workflows/build.yaml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 27516344..1a950225 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -171,28 +171,6 @@ jobs: - -######################################################################################### -## Pack for a fresh install (USB flashing) -######################################################################################### - pack-for-fresh-install: - # creates old style binaries for fresh installation (backward compatible to wiki) - runs-on: ubuntu-latest - needs: build - - steps: - - uses: actions/checkout@v2 - - - name: Get generated files from cache - uses: actions/cache@v3 - with: - path: | - ./code/.pio/build/esp32cam/firmware.bin - ./code/.pio/build/esp32cam/partitions.bin - ./code/.pio/build/esp32cam/bootloader.bin - ./sd-card/html/version.txt - key: ${{ github.run_number }} - ######################################################################################### ## Prepare and create release ######################################################################################### From 2e4023cfd6b90baafd0c8247a81c76db847108f4 Mon Sep 17 00:00:00 2001 From: Frank Haverland Date: Sun, 25 Sep 2022 18:46:35 +0200 Subject: [PATCH 2/9] pack for fresh install now cleaner part of build --- .github/workflows/build.yaml | 76 ++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1a950225..338e6e8f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -171,12 +171,63 @@ jobs: + +######################################################################################### +## Pack for a fresh install (USB flashing) (initial_esp32_setup) +######################################################################################### + pack-for-fresh-install: + # creates old style binaries for fresh installation (backward compatible to wiki) + runs-on: ubuntu-latest + needs: build + + steps: + - uses: actions/checkout@v2 + + - name: Get generated files from cache + uses: actions/cache@v3 + with: + path: | + ./code/.pio/build/esp32cam/firmware.bin + ./code/.pio/build/esp32cam/partitions.bin + ./code/.pio/build/esp32cam/bootloader.bin + ./sd-card/html/version.txt + key: ${{ github.run_number }} + + - name: Prepare artifacts for release + run: | + mkdir -p firmware + rm -rf firmware/* + mkdir -p release + # copy builds to firmware folder + cp -f "./code/.pio/build/esp32cam/firmware.bin" "firmware/firmware.bin" + cp -f "./code/.pio/build/esp32cam/bootloader.bin" "firmware/bootloader.bin" + cp -f "./code/.pio/build/esp32cam/partitions.bin" "firmware/partitions.bin" + zip -r ./firmware/sd-card.zip sd-card + cd ../firmware + zip -r ../release/initial_esp32_setup.zip . + + - name: Upload initial_esp32_setup.zip artifact (Firmware + Bootloader + Partitions + Web UI) + uses: actions/upload-artifact@v3 + with: + name: "initial_esp32_setup__${{ github.ref_name }}_(${{ steps.vars.outputs.sha_short }})" + path: ./firmware + + - name: Store generated files in cache + uses: actions/cache@v3 + with: + path: | + ./firmware + key: ${{ github.run_number }}-pack-for-fresh-install + + + + ######################################################################################### ## Prepare and create release ######################################################################################### release: runs-on: ubuntu-latest - needs: pack-for-OTA-v2 + needs: [pack-for-OTA-v2, pack-for-fresh-install] steps: - uses: actions/checkout@v2 @@ -192,6 +243,14 @@ jobs: ./dist key: ${{ github.run_number }}-pack-for-OTA-v2 + # import cached artifacts from pack-for-fresh-install + - name: Get generated files from cache + uses: actions/cache@v3 + with: + path: | + ./firmware + key: ${{ github.run_number }}-pack-for-fresh-install + - name: Set Variables id: vars run: | @@ -200,28 +259,13 @@ jobs: - name: Prepare artifacts for release run: | - ls -la ./dist - mkdir -p firmware - rm -rf firmware/* mkdir -p release - # copy builds to firmware folder - cp -f "./code/.pio/build/esp32cam/firmware.bin" "firmware/firmware.bin" - cp -f "./code/.pio/build/esp32cam/bootloader.bin" "firmware/bootloader.bin" - cp -f "./code/.pio/build/esp32cam/partitions.bin" "firmware/partitions.bin" - zip -r ./firmware/sd-card.zip sd-card # create a update.zip like "update__rolling" cd ./dist zip -r ../release/update.zip . cd ../firmware zip -r ../release/initial_esp32_setup.zip . - - - name: Upload initial_esp32_setup.zip artifact (Firmware + Bootloader + Partitions + Web UI) - uses: actions/upload-artifact@v3 - with: - name: "initial_esp32_setup__${{ github.ref_name }}_(${{ steps.vars.outputs.sha_short }})" - path: ./firmware - # extract the version used in next step - id: get_version From 5e93f1e8bd97d2eaa8eb78a20ff897b072b9d8e6 Mon Sep 17 00:00:00 2001 From: Frank Haverland Date: Sun, 25 Sep 2022 18:59:48 +0200 Subject: [PATCH 3/9] use caches from changer --- .github/workflows/build.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 338e6e8f..59408d0b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -240,6 +240,13 @@ jobs: ./code/.pio/build/esp32cam/partitions.bin ./code/.pio/build/esp32cam/bootloader.bin ./sd-card/html/version.txt + key: ${{ github.run_number }} + + # import the changes from + - name: Get generated files from cache + uses: actions/cache@v3 + with: + path: | ./dist key: ${{ github.run_number }}-pack-for-OTA-v2 From 452339a1f01a00a215402c9a8d98ab5eb5089e27 Mon Sep 17 00:00:00 2001 From: Frank Haverland Date: Sun, 25 Sep 2022 19:05:16 +0200 Subject: [PATCH 4/9] fix indent --- .github/workflows/build.yaml | 158 +++++++++++++++++------------------ 1 file changed, 77 insertions(+), 81 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 59408d0b..ff1e0e20 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -162,10 +162,6 @@ jobs: uses: actions/cache@v3 with: path: | - ./code/.pio/build/esp32cam/firmware.bin - ./code/.pio/build/esp32cam/partitions.bin - ./code/.pio/build/esp32cam/bootloader.bin - ./sd-card/html/version.txt ./dist key: ${{ github.run_number }}-pack-for-OTA-v2 @@ -230,90 +226,90 @@ jobs: needs: [pack-for-OTA-v2, pack-for-fresh-install] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: Get generated files from cache - uses: actions/cache@v3 - with: - path: | - ./code/.pio/build/esp32cam/firmware.bin - ./code/.pio/build/esp32cam/partitions.bin - ./code/.pio/build/esp32cam/bootloader.bin - ./sd-card/html/version.txt - key: ${{ github.run_number }} + - name: Get generated files from cache + uses: actions/cache@v3 + with: + path: | + ./code/.pio/build/esp32cam/firmware.bin + ./code/.pio/build/esp32cam/partitions.bin + ./code/.pio/build/esp32cam/bootloader.bin + ./sd-card/html/version.txt + key: ${{ github.run_number }} - # import the changes from - - name: Get generated files from cache - uses: actions/cache@v3 - with: - path: | - ./dist - key: ${{ github.run_number }}-pack-for-OTA-v2 + # import the changes from + - name: Get generated files from cache + uses: actions/cache@v3 + with: + path: | + ./dist + key: ${{ github.run_number }}-pack-for-OTA-v2 - # import cached artifacts from pack-for-fresh-install - - name: Get generated files from cache - uses: actions/cache@v3 - with: - path: | - ./firmware - key: ${{ github.run_number }}-pack-for-fresh-install + # import cached artifacts from pack-for-fresh-install + - name: Get generated files from cache + uses: actions/cache@v3 + with: + path: | + ./firmware + key: ${{ github.run_number }}-pack-for-fresh-install - - name: Set Variables - id: vars - run: | - echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Set Variables + id: vars + run: | + echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" - - name: Prepare artifacts for release - run: | - mkdir -p release - # create a update.zip like "update__rolling" - cd ./dist - zip -r ../release/update.zip . - cd ../firmware - zip -r ../release/initial_esp32_setup.zip . - - - # extract the version used in next step - - id: get_version - if: startsWith(github.ref, 'refs/tags/') - uses: battila7/get-version-action@v2 + - name: Prepare artifacts for release + run: | + mkdir -p release + # create a update.zip like "update__rolling" + cd ./dist + zip -r ../release/update.zip . + cd ../firmware + zip -r ../release/initial_esp32_setup.zip . + + + # extract the version used in next step + - id: get_version + if: startsWith(github.ref, 'refs/tags/') + uses: battila7/get-version-action@v2 - # the changelog [unreleased] will now be changed to the release version - - name: Update changelog - uses: thomaseizinger/keep-a-changelog-new-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - changelogPath: Changelog.md - version: ${{ steps.get_version.outputs.version-without-v }} - - # the release notes will be extracted from changelog - - name: Extract release notes - id: extract-release-notes - if: startsWith(github.ref, 'refs/tags/') - uses: ffurrer2/extract-release-notes@v1 - with: - changelog_file: Changelog.md + # the changelog [unreleased] will now be changed to the release version + - name: Update changelog + uses: thomaseizinger/keep-a-changelog-new-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + changelogPath: Changelog.md + version: ${{ steps.get_version.outputs.version-without-v }} + # the release notes will be extracted from changelog + - name: Extract release notes + id: extract-release-notes + if: startsWith(github.ref, 'refs/tags/') + uses: ffurrer2/extract-release-notes@v1 + with: + changelog_file: Changelog.md + - # Releases should only be created on master by tagging the last commit. - # all artifacts in firmware folder pushed to the release - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - name: ${{ steps.get_version.outputs.version-without-v }} - body: ${{ steps.extract-release-notes.outputs.release_notes }} - files: | - release/* - + # Releases should only be created on master by tagging the last commit. + # all artifacts in firmware folder pushed to the release + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + name: ${{ steps.get_version.outputs.version-without-v }} + body: ${{ steps.extract-release-notes.outputs.release_notes }} + files: | + release/* + - # Commit&Push Changelog to master branch. Must be manually merged back to rolling - - name: Commit changes and push changes - if: startsWith(github.ref, 'refs/tags/') - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git add Changelog.md - git commit Changelog.md -m "Update Changelog.md for ${{github.event.inputs.versionIncrement}} release" - git push origin HEAD:master + # Commit&Push Changelog to master branch. Must be manually merged back to rolling + - name: Commit changes and push changes + if: startsWith(github.ref, 'refs/tags/') + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git add Changelog.md + git commit Changelog.md -m "Update Changelog.md for ${{github.event.inputs.versionIncrement}} release" + git push origin HEAD:master From a3a46fb0bba030dc9d01732c29a1cdf7b33338c3 Mon Sep 17 00:00:00 2001 From: Frank Haverland Date: Sun, 25 Sep 2022 19:22:13 +0200 Subject: [PATCH 5/9] remove release creation artefact from pack-for-fresh-install --- .github/workflows/build.yaml | 5 +-- Changelog.md | 81 ++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ff1e0e20..6d6c991c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -199,9 +199,8 @@ jobs: cp -f "./code/.pio/build/esp32cam/bootloader.bin" "firmware/bootloader.bin" cp -f "./code/.pio/build/esp32cam/partitions.bin" "firmware/partitions.bin" zip -r ./firmware/sd-card.zip sd-card - cd ../firmware - zip -r ../release/initial_esp32_setup.zip . - + cd ./firmware + - name: Upload initial_esp32_setup.zip artifact (Firmware + Bootloader + Partitions + Web UI) uses: actions/upload-artifact@v3 with: diff --git a/Changelog.md b/Changelog.md index 50a27a06..8127b7ac 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,12 +5,87 @@ ### Added - automatic release creation -- newest firmware of rolling branch now uploaded to -- \#1068 new safer and easier update mechanismn. Use only the update.zip of the release for firmware, html and new models. +- newest firmware of rolling branch now uploaded to (developers only) +- \#1068 New update mechanism: + - handling of all files (zip, tfl, tflite, bin) within in one updload + - using the update.zip from release page + - status (upload, processing, ...) displayed + - auto suggestion for reboot (or not in case of web ui update only) +### Changed +- Integrated version info better into the html (main page, logfile) +- Updated menue +- Update tflite, esp32-cam-master, esp-nn (as of today 20220924) ### Fixed -- \#1029 wrong change of checkDigitConsistency now working like releases before 11.3.1 +- \#1029 wrong change of checkDigitConsistency now working like releases before 11.3.1 +- Spelling corrections (**[cristianmitran](https://github.com/cristianmitran)**) +- Integrated version info better into the html (main page, logfile) + + +### Removed +- Remove `/firmware` from GitHub. + - If you want to get the latest firmware and html files, please download from the automated (build action)[https://github.com/jomjol/AI-on-the-edge-device/actions] or (release page)[https://github.com/jomjol/AI-on-the-edge-device/releases] + +## [11.3.1] - (2022-09-17) +Intermediate Digits + +- **ATTENTION**: + - first update the 'firmware.bin' and ensure that the new version is running + + - Only afterwards update the 'html.zip' + + - Otherwise the downwards compatibility of the new counter clockwise feature is not given and you end in a reboot loop, that needs manual flashing! + + + +- **NEW v11.3.1**: corrected corrupted asset `firmware.bin` +- Increased precision (more than 6-7 digits) +- Implements Counter Clockwise Analog Pointers +- Improved post processing algorithm +- Debugging: intensive use of testcases +- MQTT: improved handling, extended logging, automated reconnect +- HTML: Backup Option for Configuration +- HTML: Improved Reboot +- HTML: Update WebUI (Reboot, Infos, CPU Temp, RSSI) +- This version is largely also based on the work of **[caco3](https://github.com/caco3)**, **[adellafave](https://github.com/adellafave)**, **[haverland](https://github.com/haverland)**, **[stefanbode](https://github.com/stefanbode)**, **[PLCHome](https://github.com/PLCHome)** + +## [11.2.0] - (2022-08-28) +Intermediate Digits + +- Updated Tensorflow / TFlite to newest tflite (version as of 2022-07-27) +- Updated analog neural network file (`ana-cont_11.3.0_s2.tflite` - default, `ana-class100_0120_s1_q.tflite`) +- Updated digital neural network file (`dig-cont_0570_s3.tflite` - default, `dig-class100_0120_s2_q.tflite`) + +- Added automated filtering of tflite-file in the graphical configuration (thanks to @**[caco3](https://github.com/caco3)**) +- Updated consistency algorithm & test cases +- HTML: added favicon and system name, Improved reboot dialog (thanks to @**[caco3](https://github.com/caco3)**) + +## [11.1.1] - (2022-08-22) +Intermediate Digits + +- New and improved consistency check (especially with analog and digital counters mixed) +- Bug Fix: digital counter algorithm + +## [11.0.1] - (2022-08-18) +Intermediate Digits + +- **NEW v11.0.1**: Bug Fix InfluxDB configuration (only update of html.zip necessary) + +- Implementation of new CNN types to detect intermediate values of digits with rolling numbers + + - By default the old algo (0, 1, ..., 9, "N") is active (due to the limited types of digits trained so far) + - Activation can be done by selection a tflite file with the new trained model in the 'config.ini' + - **Details can be found in the [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Neural-Network-Types)** (different types, trained image types, naming convention) + +- Updated neural network files (and adaption to new naming convention) + +- Published a tool to download and combine log files - **Thanks to ** + + - Files see ['/tools/logfile-tool'](tbd), How-to see [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Gasmeter-Log-Downloader) + +- Bug Fix: InfluxDB enabling in grahic configuration + ## [10.6.2] - (2022-07-24) From b38c940dd32d5a7cd8f9407371373f75fecf98bf Mon Sep 17 00:00:00 2001 From: Frank Haverland Date: Sun, 25 Sep 2022 19:33:58 +0200 Subject: [PATCH 6/9] run release only on tagging --- .github/workflows/build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6d6c991c..7ab99a0d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -223,7 +223,8 @@ jobs: release: runs-on: ubuntu-latest needs: [pack-for-OTA-v2, pack-for-fresh-install] - + if: startsWith(github.ref, 'refs/tags/') + steps: - uses: actions/checkout@v2 From 228a87038ec530bfbe332bdbd5da94c335e994bc Mon Sep 17 00:00:00 2001 From: Frank Haverland Date: Sun, 25 Sep 2022 19:43:30 +0200 Subject: [PATCH 7/9] Link to the installtion guide in firmware-Readme. It will packed into the initial-setup.zip --- .github/workflows/build.yaml | 2 +- firmware/README.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7ab99a0d..29f866d2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -192,7 +192,7 @@ jobs: - name: Prepare artifacts for release run: | mkdir -p firmware - rm -rf firmware/* + rm -rf firmware/*.zip mkdir -p release # copy builds to firmware folder cp -f "./code/.pio/build/esp32cam/firmware.bin" "firmware/firmware.bin" diff --git a/firmware/README.md b/firmware/README.md index e09cc8e1..bf3f5130 100644 --- a/firmware/README.md +++ b/firmware/README.md @@ -1,2 +1,6 @@ # Firmware The firmware got moved to the [Release page](https://github.com/jomjol/AI-on-the-edge-device/releases). + +# Installation Guide + +You find the complete installation guide at From 2239ab90192ae4308bacc573537b7a26905420ef Mon Sep 17 00:00:00 2001 From: jomjol <30766535+jomjol@users.noreply.github.com> Date: Mon, 26 Sep 2022 18:19:09 +0200 Subject: [PATCH 8/9] Update server_ota.cpp --- code/components/jomjol_fileserver_ota/server_ota.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/components/jomjol_fileserver_ota/server_ota.cpp b/code/components/jomjol_fileserver_ota/server_ota.cpp index eb87c199..961bee16 100644 --- a/code/components/jomjol_fileserver_ota/server_ota.cpp +++ b/code/components/jomjol_fileserver_ota/server_ota.cpp @@ -303,7 +303,7 @@ esp_err_t handler_ota_update(httpd_req_t *req) LogFile.WriteToFile("handler_ota_update"); char _query[200]; - char _filename[30]; + char _filename[100]; char _valuechar[30]; std::string fn = "/sdcard/firmware/"; bool _file_del = false; @@ -319,12 +319,12 @@ esp_err_t handler_ota_update(httpd_req_t *req) _task = std::string(_valuechar); } - if (httpd_query_key_value(_query, "file", _filename, 30) == ESP_OK) + if (httpd_query_key_value(_query, "file", _filename, 100) == ESP_OK) { fn.append(_filename); printf("File: "); printf(fn.c_str()); printf("\n"); } - if (httpd_query_key_value(_query, "delete", _filename, 30) == ESP_OK) + if (httpd_query_key_value(_query, "delete", _filename, 100) == ESP_OK) { fn.append(_filename); _file_del = true; From aa72cab84f1949d32dee4e2d8463b049b2c07b5d Mon Sep 17 00:00:00 2001 From: Frank Haverland Date: Mon, 26 Sep 2022 19:57:59 +0200 Subject: [PATCH 9/9] reduce SCRATCH_BUFSIZE to fix upload errors --- code/components/jomjol_fileserver_ota/server_file.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/components/jomjol_fileserver_ota/server_file.cpp b/code/components/jomjol_fileserver_ota/server_file.cpp index 07487c73..5bdd89c2 100644 --- a/code/components/jomjol_fileserver_ota/server_file.cpp +++ b/code/components/jomjol_fileserver_ota/server_file.cpp @@ -54,7 +54,7 @@ extern "C" { /* Scratch buffer size */ -#define SCRATCH_BUFSIZE 8192 +#define SCRATCH_BUFSIZE 4096 struct file_server_data { /* Base path of file storage */