v11.2.0

Beautify restart

.gitignore

@@ -4,6 +4,7 @@
 .code-workspace
 /sd-card/htm./.vscode/
 /code/build
+/sd-card/html/debug/
 
 CMakeLists.txt.user
 CMakeCache.txt
@@ -15,3 +16,5 @@ install_manifest.txt
 compile_commands.json
 CTestTestfile.cmake
 _deps
+code/edgeAI.code-workspace
+.DS_Store

Changelog.md

@@ -1,6 +1,447 @@
 # Versions
 
+##### 10.6.2 - Stability Increase (2022-07-24)
+
+- **NEW 10.6.2**: ignore hidden files in model selection (configuration page) 
+
+- **NEW 10.6.1**: Revoke esp32cam & tflite update
+
+- **NEW 10.6.1**: Bug Fix: tflite-filename with ".", HTML spelling error
+
+- IndluxDB: direct injection into InfluxDB - thanks to **[wetneb](https://github.com/wetneb)**
+
+- MQTT: implemented "Retain Flag" and extend with absolute Change (in addition to rate)
+
+- `config.ini`: removal of modelsize (readout from tflite)
+
+- Updated analog neural network file (`ana1000s2.tflite`) & digital neural network file (`dig1400s2q.tflite`)
+
+- TFMicro/Lite: Update (espressif Version 20220716)
+
+- Updated esp32cam (v20220716)
+
+- ESP-IDF: Update to 4.4
+
+- Internal update (CNN algorithm optimizations, reparation for new neural network type)
+
+- Bug Fix: no time with fixed IP, Postprocessing, MQTT
+
+  
+
+##### 10.5.2 - Stability Increase (2022-02-22)
+
+- NEW 10.5.2: Bug Fix: wrong `firmware.bin` (no rate update)
+- NEW 10.5.1: Bug Fix: wrong return value, rate value & PreValue status, HTML: SSID & IP were not displayed 
+- MQTT: changed wifi naming to "wifiRSSI"
+- HTML: check selectable values for consistency
+- Refactoring of check postprocessing consistency (e.g. max rate, negative rate, ...)
+- Bug Fix: corrected error in "Check Consistency Increase"
+
+
+
+##### 10.4.0 - Stability Increase (2022-02-12)
+
+- Graphical configuration: select available neural network files (*.tfl, *.tflite) from drop down menu
+- OTA-update: add option to upload tfl / tflite files to the correct location (`/config/`)
+  - In the future the new files will also be copied to the `firmware` directory of the repository
+- Added Wifi RSSI to MQTT information
+- Updated analog neural network file (`ana-s3-q-20220105.tflite`)
+- Updated digital neural network file (`dig-s1-q-20220102.tflite`)
+- Updated build environment to `Espressif 3.5.0`
+
+
+
+##### 10.3.0 - Stability Increase (2022-01-29)
+
+- Implemented LED flash dimming (`LEDIntensity`). 
+  Remark: as auto illumination in the camera is used, this is rather for energy saving. It will not help reducing reflections
+- Additional camera parameters: saturation, contrast (although not too much impact yet)
+- Some readings will have removable "N"s that can not be removed automatically and are handled with an "error" --> no return value in the field "value" anymore (still reported back via field "raw value")
+- Updated esp32 camera hardware driver
+- Bug fix: MQTT, HTML improvements
+
+**ATTENTION:  The new ESP32 camera hardware driver is much more stable on newer OV2640 versions (no or much less reboots) but seems to be not fully compatible with older versions.**
+
+* If you have problem with stalled systems you can try the following
+  - Update the parameter `ImageQuality` to `12` instead of current value `5` (manually in the `config.ini`)
+
+  - If this is not helping, you might need to update your hardware or stay with version 9.2
+
+##### 10.2.0 - Stability Increase (2022-01-14)
+
+- Due to the updated camera driver, the image looks different and a new setup might be needed
+
+  - Update reference image
+  - Update Alignment marks
+
+- Reduce reboot due to camera problems
+
+- Update esp32-camera to new version (master as of 2022-01-09)
+
+  
+
+##### 10.1.1 - Stability Increase (2022-01-12)
+
+- Bug Fix MQTT problem
+- Issue:
+  - Changing from v9.x to 10.x the MQTT-parameter "Topic" was renamed into "MainTopic" to address multiple number meters. This renaming should have been done automatically in the background within the graphical configuration, but was not working. Instead the parameter "Topic" was deleted and "MainTopic" was set to disabled and "undefined".
+- ToDo
+  - Update the `html.zip`
+  - If old `config.ini` available: copy it to `/config`, open the graphical configuration and save it again.
+  - If old `config.ini` not available: reset the parameter "MainTopic" within the `config.ini` manually
+  - Reboot
+
+##### 10.1.0 - Stability Increase (2022-01-09)
+
+- Reduce ESP32 frequency to 160MHz
+
+- Update tflite (new source: https://github.com/espressif/tflite-micro-esp-examples)
+
+- Update analog neural network (ana-s3-q-20220105.tflite)
+
+- Update digital neural network (dig-s1-q-20220102.tflite)
+
+- Increased web-server buffers
+- bug fix: compiler compatibility
+
+##### 10.0.2 - Stability Increase (2022-01-01)
+
+- NEW v10.0.2: Corrected JSON error
+
+- Updated compiler toolchain to ESP-IDF 4.3
+
+- Removal of memory leak
+
+- Improved error handling during startup (check PSRAM and camera with remark in logfile)
+
+- MQTT: implemented raw value additionally, removal of regex contrain
+
+- Normalized Parameter ``MaxRateValue``  to "change per minute" 
+
+- HTML: improved input handling
+
+- Corrected error handling: in case of error the old value, rate, timestamp are not transmitted any more
+
+  
+
+##### 9.2.0 - External Illumination (2021-12-02)
+
+- Direct JSON access: ``http://IP-ADRESS/json`` 
+- Error message in log file in case camera error during startup
+- Upgrade analog CNN to v9.1.0
+- Upgrade digital CNN to v13.3.0 (added new images)
+- html: support of different ports
+
+##### 9.1.1 - External Illumination (2021-11-16)
+
+- NEW 9.1.1 bug fix: LED implemenetation
+- External LEDs: change control mode (resolve bug with more than 2 LEDs)
+- Additional info into log file
+- Bug fix: decimal shift, html, log file
+
+##### 9.0.0 - External Illumination (2021-10-23)
+
+* Implementation of external illumination to adjust positioning, brightness and color of the illumination now set individually
+  * Technical details can be found in the wiki: https://github.com/jomjol/AI-on-the-edge-device/wiki/External-LED
+    <img src="https://raw.githubusercontent.com/jomjol/ai-on-the-edge-device/master/images/intern_vs_external.jpg" width="500">
+* New housing published for external LEDs and small clearing: https://www.thingiverse.com/thing:5028229
+
+
+
+##### 8.5.0 - Multi Meter Support (2021-10-07)
+
+* Upgrade digital CNN to v13.1.0 (added new images)
+* bug fix: wlan password with space, double digit output
+
+##### 8.4.0 - Multi Meter Support (2021-09-25)
+
+* License change (remove MIT license, remark see below)
+
+* html: show hostname in title and main page
+
+* configuration: 
+
+  * moved setting `ExtendedResolution` to individual number settings
+  * New parameter `IgnoreLeadingNaN` (delete leading NaN's specifically)
+  * **ATTENTION**: update of the `config.ini` needed (open, adjust `ExtendedResolution`, save)
+
+* Bug fixing (html, images of recognized numbers)
+
+  
+
+### **ATTENTION: LICENSE CHANGE - removal of MIT License.** 
+
+- Currently no licence published - copyright belongs to author
+- If you are interested in a commercial usage or dedicated versions please contact the developer
+  - no limits to private usage
+
+
+
+##### 8.3.0 - Multi Meter Support (2021-09-12)
+
+* Upgrade digital CNN to v12.1.0 (added new images)
+* Dedicated NaN handling, internal refactoring (CNN-Handling)
+* HTML: confirmation after config.ini update
+* Bug fixing
+
+##### 8.2.0 - Multi Meter Support (2021-08-24)
+
+* Improve server responsiveness
+* Flow status and prevalue status in overview
+* Improved prevalue handling 
+
+##### 8.1.0 - Multi Meter Support (2021-08-12)
+
+* GPIO: using the general mqtt main topic for GPIO
+
+* Upgrade digital CNN to v12.0.0  (added new images)
+* Update tfmicro to new master (2021-08-07)
+* Bug fix: remove text in mqtt value, remove connect limit in wlan reconnet
+
+##### 8.0.5 - Multi Meter Support (2021-08-01)
+
+* NEW 8.0.5: bug fix: saving prevalue
+* NEW 8.0.4: bug fix: load config.ini after upgrade
+* NEW 8.0.3: bug fix: reboot during `config.ini` handling, html error
+* NEW 8.0.2: saving roundes prevalue, bug fix html server
+* NEW 8.0.1: bug fix: html handling of parameter `FixedExposure` and `ImageSize`
+* Dual / multi meter support (more than 1 number to be recognized)
+  This is implemented with the feature "number" on the ROI definition as well as selected options
+* MQTT: standardization of the naming - including new topics (`json`,  `freeMem `, `uptime`)c
+* Preparation for extended GPIO support (thanks to Zwerk2k) - not tested and fully functional yet
+* Bug fixing: html server, memory leak, MQTT connect, hostname, turn of flash LED
+
+<span style="color: red;">**ATTENTION: the configuration and prevalue files are modified automatically and will not be backward compatible!**</span> 
+
+##### 7.1.2 MQTT-Update - (2021-06-17)
+
+* NEW: 7.1.2: bug fix setting hostname, Flash-LED not off during reboot
+
+* NEW: 7.1.1: bug fix wlan password with "="  (again)
+
+* MQTT error message: changes "no error", send retain flag
+
+* Update wlan handling to esp-idf 4.1
+
+* Upgrade digital CNN to v8.7.0  (added new images)
+
+* Bug fix: MQTT, WLAN, LED-Controll, GPIO usage, fixed IP, calculation flow rate
+
+  
+
+##### 7.0.1 MQTT-Update - (2021-05-13)
+
+* NEW: 7.0.1: bug fix wlan password with "=" 
+
+* Upgrade digital CNN to v8.5.0  (added new images)
+
+* New MQTT topics: flow rate (units/minute), time stamp (last correct read readout)
+
+* Update MQTT/Error topic to " " in case no error (instead of empty string)
+
+* Portrait or landscape image orientation in rotated image (avoid cropping)
+
+##### 6.7.2 Image Processing in Memory - (2021-05-01)
+
+* NEW 6.7.2: Updated html for setup modus - remove reboot on edit configuration)
+
+* NEW 6.7.1: Improved stability of camera (back to v6.6.1) - remove black strips and areas
+
+* Upgrade digital CNN to v8.3.0  (added new type of digits)
+
+* Internal update: TFlite (v2.5), esp32cam, startup sequence
+
+* Rollback to espressif v2.1.0, as v3.2.0 shows unstable reboot
+
+* Bugfix: WLan-passwords, reset of hostname
+
+
+##### 6.6.1 Image Processing in Memory - (2021-04-05)
+
+* NEW 6.6.1: failed SD card initialization indicated by fast blinking LED at startup
+* Improved SD-card handling (increase compatibility with more type of cards)
+
+##### 6.5.0 Image Processing in Memory - (2021-03-25)
+
+* Upgrade digital CNN to v8.2.0  (added new type of digits)
+* Supporting alignment structures in ROI definition
+* Bug fixing: definition of  hostname in `config.ini`
+
+##### 6.4.0 Image Processing in Memory - (2021-03-20)
+
+* Additional alignment marks for settings the ROIs (analog and digit)
+* Upgrade analog CNN to v7.0.0 (added new type of pointer)
+
+##### 6.3.1 Image Processing in Memory - (2021-03-16)
+
+* NEW: 6.3.1: bug fixing in initial edit reference image and `config.ini` (Spelling error in `InitialRotate`)
+* Initial setup mode: bug fixing, error correction
+* Bug-fixing
+
+##### 6.2.2 Image Processing in Memory - (2021-03-10)
+
+* NEW 6.2.2: bug fixing
+* NEW 6.2.1: Changed brightness and contrast to default if not enabled (resolves to bright images)
+* Determination of fixed illumination settings during startup - speed up of 5s in each run
+* Update digital CNN to v8.1.1 (additional digital images trained)
+* Extended error message in MQTT error message
+
+
+* Image brightness is now adjustable 
+
+
+* Bug fixing: minor topics 
+
+
+##### 6.1.0 Image Processing in Memory - (2021-01-20)
+
+* Disabling of analog / digital counters in configuration 
+* Improved Alignment Algorithm (`AlignmentAlgo`  = `Default`,  `Accurate` , `Fast`)
+* Analog counters: `ExtendedResolution` (last digit is extended by sub comma value of CNN)
+* `config.ini`: additional parameter `hostname`  (additional to wlan.ini)
+* Switching of GPIO12/13 via http-interface: `/GPIO?GPIO=12&Status=high/low`
+* Bug fixing: html configuration page, wlan password ("=" now possible)
+
+##### 6.0.0 Image Processing in Memory - (2021-01-02)
+
+* **Major change**: image processing fully in memory - no need of SD card buffer anymore
+
+  * Need to limit camera resolution to VGA (due to memory limits)
+* MQTT: Last Will Testament (LWT) implemented: "connection lost" in case of connection lost to `TopicError`
+* Disabled `CheckDigitIncreaseConsistency` in default configuration - must now be explicit enabled if needed
+* Update digital CNN to v7.2.1 (additional digital images trained) 
+* Setting of arbitrary time server in `config.ini`
+* Option for fixed IP-, DNS-Settings in `wlan.ini`
+* Increased stability (internal image and camera handling)
+* Bug fixing: edit digits, handling PreValue, html-bugs
+
+
+
+##### 5.0.0 Setup Modus - (2020-12-06)
+
+* Implementation of initial setup modus for fresh installation
+
+* Code restructuring (full compatibility between pure ESP-IDF and Platformio w/ espressif)
+
+  
+
+##### 4.1.1 Configuration editor - (2020-12-02)
+
+* Bug fixing: internal improvement of file handling (reduce not responding)
+
+
+##### 4.1.0 Configuration editor - (2020-11-30)
+
+* Implementation of configuration editor (including basic and expert mode)
+
+* Adjustable time zone to adjust to local time setting (incl. daylight saving time)
+
+* MQTT: additional topic for error reporting
+
+* standardized access to current logfile via `http://IP-ADRESS/logfileact`
+
+* Update digital CNN to v7.2.0, analog CNN to 6.3.0
+
+* Bug fixing: truncation error,  CheckDigitConsistency & PreValue implementation
+
+
+
+##### 4.0.0 Tflite Core - (2020-11-15)
+
+* Implementation of rolling log-files
+
+* Update Tflite-Core to master@20201108 (v2.4)
+
+* Bug-fixing for reducing reboots
+
+  
+
+##### 3.1.0 MQTT-Client - (2020-10-26)
+
+* Update digital CNN to v6.5.0 and HTML (Info to hostname, IP, ssid)
+
+* New implementation of "checkDigitConsistency" also for digits
+* MQTT-Adapter: user and password for sign in MQTT-Broker
+
+##### 3.0.0 MQTT-Client  (2020-10-14)
+
+* Implementation of MQTT Client
+* Improved Version Control
+* bug-fixing
+
+
+
+##### 2.2.1 Version Control  (2020-09-27)
+
+* Bug-Fixing (hostname in wlan.ini and error handling inside flow)
+
+
+##### 2.2.0 Version Control  (2020-09-27)
+
+* Integrated automated versioning system (menu: SYSTEM --> INFO)
+* Update Build-System to PlatformIO - Espressif 32 v2.0.0 (ESP-IDF 4.1)
+
+
+##### 2.1.0 Decimal Shift, Chrome & Edge  (2020-09-25)
+
+* Implementation of Decimal Shift
+
+* Update default CNN for digits to v6.4.0
+
+* Improvement HTML
+
+* Support for Chrome and Edge
+
+* Reduce logging to minimum - extended logging on demand
+
+* Implementation of hostname in wlan.ini (`hostname = "HOSTNAME")`
+
+* Bug fixing, code corrections
+
+
+##### 2.0.0 Layout update  (2020-09-12)
+
+  * Update to **new and modern layout**
+  * Support for Chrome improved
+  * Improved robustness: improved error handling in auto flow reduces spontaneous reboots
+  * File server: Option for "DELETE ALL"
+  * WLan: support of spaces in SSID and password
+  * Reference Image: Option for mirror image, option for image update on the fly
+  * additional parameter in `wasserzaehler.html?noerror=true`  to suppress an potential error message
+  * bug fixing
+
+
+
+##### 1.1.3 (2020-09-09)
+
+* **Bug in configuration of analog ROIs corrected** - correction in v.1.0.2 did not work properly
+* Improved update page for the web server (`/html` can be updated via a zip-file, which is provided in `/firmware/html.zip`)
+* Improved Chrome support
+
+##### 1.1.0 (2020-09-06)
+
+* Implementation of "delete complete directory"
+  **Attention: beside the `firmware.bin`, also the content of `/html` needs to be updated!**
+
+
+
+##### 1.0.2 (2020-09-06)
+
+* Bug in configuration of analog ROIs corrected
+* minor bug correction
+
+##### 1.0.1 (2020-09-05)
+
+* preValue.ini Bug corrected
+* minor bug correction
+
+##### 1.0.0 (2020-09-04)
+
+* **First usable version** - compatible to previous project (https://github.com/jomjol/water-meter-system-complete)
+* NEW: 
+  * no docker container for CNN calculation necessary
+  * web based configuration editor on board
+
 ##### 0.1.0 (2020-08-07)
 
-* Initial Version
-
+* Initial Version

FeatureRequest.md

@@ -0,0 +1,248 @@
+## Feature Requests
+
+**There are a lot of ideas for further improvements, but only limited capacity on side of the developer.** Therefore I have created this page as a collection of ideas. 
+
+1. Who ever has a new idea can put it here, so it that it is not forgotten. 
+
+2. Who ever has time, capacity and passion to support, can take any of the ideas and implement them. 
+   I will support and help where ever I can!
+   
+   
+
+____
+
+#### #29 Add favicon and use the hostname for the website
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/927
+
+#### #28 Improved error handling for ROIs
+
+* In case a ROI is out of the image, there is no error message, but a non sense image is used
+* Implement a error message for wrong configuratioin of ROI
+
+#### #27 Use Homie Spec for Mqtt binding
+
+* Use the standardized Home Protocol for the Mqtt binding 
+* https://homieiot.github.io/
+
+#### #26 Changes behaviour for "N" replacement
+
+* in case the higher digits has already increased by minium 1 - don't set the "N" to the last value, but to "0"
+* https://github.com/jomjol/AI-on-the-edge-device/issues/792
+
+
+#### #25 Trigger Measurement via MQTT
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/727
+
+
+#### #24 Show Mqtt state directly in Webserver
+
+* Show MQTT log in Web page. E.g. connection established or failed to connect...
+
+  
+
+
+#### #23 CPU Temp and Mqtt values
+
+* Show the CPU Temp directly in Webpage. Also add the value to MQTT sending
+
+  
+
+#### #22 Direct hint to the different neural network files in the other repositories
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/644
+
+  
+
+#### #21 Extended "CheckDigitalConsistency" Logik
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/590
+
+  
+
+#### #20 Deep sleep and push mode
+
+* Let the device be normally in deep sleep state, and wake it up periodically to collect data and push it via MQTT or HTTP post.
+* Support ESP-NOW to reduce the overhead of connecting to wifi and mqtt 
+* the above should enable battery powered applications
+  
+
+#### #19 Extended log informations
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/580
+
+  
+
+#### ~~#18 Document WLAN-strength in web page~~
+
+* ~~https://github.com/jomjol/AI-on-the-edge-device/issues/563~~
+
+
+
+#### ~~#17 Direct InfluxDB connection~~
+
+* ~~Done in v10.6.0~~
+
+
+#### #16 Serial Communication
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/512
+* Send the readout value via RX/TX interface with a dedicated TAG
+* Make dedicated communication FlowModule
+* Modification of RX/TX communication
+* Configuration interfache
+
+
+#### #15 Calibration for FishEye image
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/507
+
+1.  The development of such a correction algorithm with the libraries, that are available for the ESP32 environment.
+2. New module for integration of the flow into the image processing flow.
+3. Extension of the configuration (config.ini) and html-pages
+4. Parameter adjustment and testing for every different fish-eye module
+5. Maintenance for further updates / modules, ...
+
+
+
+#### #14 Backup and restore option for configuration
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/459
+
+* Implement a zip file compression for store and restore
+
+* Update the html to handle it
+
+  
+
+#### #13 Manage non linear gauge without CNN re-training
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/443
+
+* Implement a look up table for non linear analog meters
+
+  
+
+#### ~~#12 Less reboots due to memory leakage~~
+
+* ~~Issue: #414 & #425  #430~~
+
+  
+
+#### #11 MQTT - configurable payload
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/344
+
+  
+
+#### #10 Improve and bug fix logging of images
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/307
+
+  
+
+#### #9 Basic auth for the UI
+
+* https://github.com/jomjol/AI-on-the-edge-device/issues/283
+
+* Implementation of an authentication mechanism.
+
+#### #8 MQTT configurable readout intervall
+
+Make the readout intervall configurable via MQTT.
+
+* Change the mqtt part to receive and process input and not only sending
+
+#### #7 Extended Error Handling
+
+Check different types of error (e.g. tflite not availabe) and generate an error on the html page.
+
+To do:
+
+* Make a list of "important" errors
+* Implement a checking algo
+* Extend the firmware and html page for the error handling
+
+#### ~~#6 Check for double ROI names~~ - implemented v8.0.0
+
+~~Check during configuration, that ROI names are unique.~~
+
+~~To do:~~
+
+* ~~Implementation of ROI name checking in html code before saving analog or digital ROIs~~
+
+  
+
+#### #5 Configurable decimal separator (point or comma) 
+
+Decimal separator configurable for different systems
+
+To do:
+
+* Implementation of decimal point into postprocessing module
+* Extension of configuration
+* Adaption of the html configuration to implement shifting
+
+
+
+#### ~~#4 Initial Shifting and Rotation~~ - implemented v7.0.0
+
+* ~~https://github.com/jomjol/AI-on-the-edge-device/issues/123~~
+
+~~Implementation of a shifting additional to the initial rotation of the raw camera input~~
+
+~~To do:~~
+
+* ~~Implementation of shifting~~
+* ~~Extension of configuration~~
+* ~~Adaption of the html configuration to implement shifting~~
+
+
+
+#### ~~#3 Allow grouping of digits to multiple reading values~~ - implemented v8.0.0
+
+* ~~https://github.com/jomjol/AI-on-the-edge-device/issues/123~~
+
+~~Implementation of two different independent readouts in one setup~~
+
+~~To do:~~
+
+* ~~Extend the configuration, setting and processing flow for two independend readouts~~
+
+
+
+
+
+____
+
+#### #2 MQTT-controll with callback 
+* https://github.com/jomjol/AI-on-the-edge-device/issues/105
+
+Extend the MQTT client to also enable callbacks for configuration setting
+
+To do:
+
+* implement callback for receiving information and override `config.ini` settings
+
+* change configuration management to handle online updates (currently changes need a restart)
+
+* think about the startup, as there the default config is loaded 
+
+  
+
+____
+
+#### ~~#1 Optional GPIO for external flash/lighting~~ - implemented (v8.0.0)
+
+* ~~https://github.com/jomjol/AI-on-the-edge-device/issues/133~~
+
+~~Implementation of an an extrnal flash / lightning through GPIOs.~~
+
+* ~~available GPIOs: 12 & 13 (currently in use for html switching)~~
+
+~~To do:~~
+
+* ~~Implementation of a software module for external light source (e.g. WS8132 LED controller, ...)~~
+* ~~Update of the camera module to use the external light instead of the internal flash light~~
+* ~~Adopt the configuration algorithm with a configurable light source~~

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2020 jomjol
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

README.md

@@ -4,12 +4,17 @@ This is an example of Artificial Intelligence (AI) calculations on a very cheap
 
 ### Details on **function**, **installation** and **configuration** can be found on the **[Wiki Page](https://github.com/jomjol/AI-on-the-edge-device/wiki)**
 
-A 3d-printable housing can be found here: https://www.thingiverse.com/thing:4571627
+A 3d-printable housing can be found here:
+  - https://www.thingiverse.com/thing:4573481 (Water Meter)
+  - https://www.thingiverse.com/thing:5028229 (Power Meter)
+  - https://www.thingiverse.com/thing:4571627 (ESP32-Cam housing only)
 
 <img src="https://raw.githubusercontent.com/jomjol/AI-on-the-edge-device/master/images/watermeter_all.jpg" width="200"><img src="https://raw.githubusercontent.com/jomjol/AI-on-the-edge-device/master/images/main.jpg" width="200"><img src="https://raw.githubusercontent.com/jomjol/AI-on-the-edge-device/master/images/size.png" width="200"> 
 
 <img src="https://raw.githubusercontent.com/jomjol/AI-on-the-edge-device/master/images/watermeter.jpg" width="600"> 
 
+<img src="https://raw.githubusercontent.com/jomjol/AI-on-the-edge-device/master/images/powermeter.jpg" width="600"> 
+
 
 
 
@@ -24,208 +29,96 @@ If you would like to support the developer with a cup of coffee you can do that
 <input type="image" src="https://www.paypalobjects.com/en_US/DK/i/btn/btn_donateCC_LG.gif" border="0" name="submit" title="PayPal - The safer, easier way to pay online!" alt="Donate with PayPal button" />
 <img alt="" border="0" src="https://www.paypal.com/en_DE/i/scr/pixel.gif" width="1" height="1" />
 </form>
+If you have any technical topics, you can file a issue in this repository. 
+
+In other cases you can contact the developer via email: <img src="https://raw.githubusercontent.com/jomjol/AI-on-the-edge-device/master/images/mail.jpg" height="25"> 
 
 
+------
 ## Change log
+**General remark:** Besides the file `firmware.bin`, typically the content of `/html` will need to be updated!
 
 ------
 
-### Known Issues
+##### 11.2.0 - Intermediate Digits (2022-08-28)
+
+- Updated Tensorflow / TFlite to newest tflite (version as of 2022-07-27)
+- Updated analog neural network file (`ana-cont_11.3.0_s2.tflite` - default, `ana-class100_0120_s1_q.tflite`)
+- Updated digital neural network file (`dig-cont_0570_s3.tflite` - default, `dig-class100_0120_s2_q.tflite`)
+
+- Added automated filtering of tflite-file in the graphical configuration (thanks to @**[caco3](https://github.com/caco3)**)
+- Updated consistency algorithm & test cases
+- HTML: added favicon and system name, Improved reboot dialog  (thanks to @**[caco3](https://github.com/caco3)**)
+
+##### 11.1.1 - Intermediate Digits (2022-08-22)
+
+- New and improved consistency check (especially with analog and digital counters mixed)
+- Bug Fix: digital counter algorithm
+
+##### 11.0.1 - Intermediate Digits (2022-08-18)
+
+- **NEW v11.0.1**: Bug Fix InfluxDB configuration (only update of html.zip necessary)
+
+- Implementation of new CNN types to detect intermediate values of digits with rolling numbers
+
+  - By default the old algo (0, 1, ..., 9, "N") is active (due to the limited types of digits trained so far)
+  - Activation can be done by selection a tflite file with the new trained model in the 'config.ini'
+  - **Details can be found in the [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Neural-Network-Types)** (different types, trained image types, naming convention)
+
+- Updated  neural network files (and adaption to new naming convention)
+
+- Published a tool to download and combine log files - **Thanks to **
+
+  - Files see ['/tools/logfile-tool'](tbd), How-to see [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Gasmeter-Log-Downloader)
+
+- Bug Fix: InfluxDB enabling in grahic configuration
+
+  
+
+## Tools
+
+* Logfile downloader and combiner (Thx to [reserve85](https://github.com/reserve85))
+  * Files see ['/tools/logfile-tool'](tbd), How-to see [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Gasmeter-Log-Downloader)
+
+
+
+## Additional Ideas
+
+There are some ideas and feature requests which are not followed currently - mainly due to capacity reasons on side of the developer. They are collected here: [FeatureRequest.md](FeatureRequest.md)
+
 
-* slow response of web server during picture analysis
-* spontaneous reboots (mostly due to html access during image processing) - self recovery implemented
 
 ------
 
-**General remark:** Beside the `firmware.bin`, typically also the content of `/html` needs to be updated!
+## History
 
+##### 10.6.2 - Stability Increase (2022-07-24)
 
+##### 9.2.0 - External Illumination (2021-12-02)
 
-##### 6.3.0 Image Processing in Memory - (2021-03-14)
+##### 8.5.0 Multi Meter Support (2021-10-07)
 
-* Initial setup mode: bug fixing, error correction
-* Bug-fixing
+##### 7.1.2 MQTT-Update - (2021-06-17)
 
-##### 6.2.2 Image Processing in Memory - (2021-03-10)
-
-* NEW 6.2.2: bug fixing
-* NEW 6.2.1: Changed brightness and contrast to default if not enabled (resolves to bright images)
-* Determination of fixed illumination settings during startup - speed up of 5s in each run
-* Update digital CNN to v8.1.1 (additional digital images trained)
-* Extended error message in MQTT error message
-
-
-* Image brightness is now adjustable 
-
-
-* Bug fixing: minor topics 
-
-
-##### 6.1.0 Image Processing in Memory - (2021-01-20)
-
-* Disabling of analog / digital counters in configuration 
-* Improved Alignment Algorithm (`AlignmentAlgo`  = `Default`,  `Accurate` , `Fast`)
-* Analog counters: `ExtendedResolution` (last digit is extended by sub comma value of CNN)
-* `config.ini`: additional parameter `hostname`  (additional to wlan.ini)
-* Switching of GPIO12/13 via http-interface: `/GPIO?GPIO=12&Status=high/low`
-* Bug fixing: html configuration page, wlan password ("=" now possible)
-
-##### 6.0.0 Image Processing in Memory - (2021-01-02)
-
-* **Major change**: image processing fully in memory - no need of SD card buffer anymore
-  
-  * Need to limit camera resolution to VGA (due to memory limits)
-  
-* MQTT: Last Will Testament (LWT) implemented: "connection lost" in case of connection lost to `TopicError`
-
-* Disabled `CheckDigitIncreaseConsistency` in default configuration - must now be explicit enabled if needed
-
-* Update digital CNN to v7.2.1 (additional digital images trained) 
-
-* Setting of arbitrary time server in `config.ini`
-
-* Option for fixed IP-, DNS-Settings in `wlan.ini`
-
-* Increased stability (internal image and camera handling)
-
-* Bug fixing: edit digits, handling PreValue, html-bugs
-
-  
+##### 6.7.2 Image Processing in Memory - (2021-05-01)
 
 ##### 5.0.0 Setup Modus - (2020-12-06)
 
-* Implementation of initial setup modus for fresh installation
-
-* Code restructuring (full compatibility between pure ESP-IDF and Platformio w/ espressif)
-  
-  
-
 ##### 4.1.1 Configuration editor - (2020-12-02)
 
-* Bug fixing: internal improvement of file handling (reduce not responding)
-
-
-##### 4.1.0 Configuration editor - (2020-11-30)
-
-* Implementation of configuration editor (including basic and expert mode)
-
-* Adjustable time zone to adjust to local time setting (incl. daylight saving time)
-
-* MQTT: additional topic for error reporting
-
-* standardized access to current logfile via `http://IP-ADRESS/logfileact`
-
-* Update digital CNN to v7.2.0, analog CNN to 6.3.0
-
-* Bug fixing: truncation error,  CheckDigitConsistency & PreValue implementation
-
-
-
 ##### 4.0.0 Tflite Core - (2020-11-15)
-* Implementation of rolling log-files
-
-* Update Tflite-Core to master@20201108 (v2.4)
-
-* Bug-fixing for reducing reboots
-  
-  
-
 ##### 3.1.0 MQTT-Client - (2020-10-26)
 
-* Update digital CNN to v6.5.0 and HTML (Info to hostname, IP, ssid)
-
-* New implementation of "checkDigitConsistency" also for digits
-* MQTT-Adapter: user and password for sign in MQTT-Broker
-
-##### 3.0.0 MQTT-Client  (2020-10-14)
-
-* Implementation of MQTT Client
-* Improved Version Control
-* bug-fixing
+##### 2.2.1 Version Control - (2020-09-27)
 
 
-
-##### 2.2.1 Version Control  (2020-09-27)
-
-* Bug-Fixing (hostname in wlan.ini and error handling inside flow)
-  
+##### 2.1.0 Decimal Shift, Chrome & Edge - (2020-09-25)
 
 
-##### 2.2.0 Version Control  (2020-09-27)
+##### 2.0.0 Layout update - (2020-09-12)
 
-* Integrated automated versioning system (menu: SYSTEM --> INFO)
-* Update Build-System to PlatformIO - Espressif 32 v2.0.0 (ESP-IDF 4.1)
-
-
-##### 2.1.0 Decimal Shift, Chrome & Edge  (2020-09-25)
-
-* Implementation of Decimal Shift
-
-* Update default CNN for digits to v6.4.0
-
-* Improvement HTML
-
-* Support for Chrome and Edge
-
-* Reduce logging to minimum - extended logging on demand
-
-* Implementation of hostname in wlan.ini (`hostname = "HOSTNAME")`
-
-* Bug fixing, code corrections
-
-
-##### 2.0.0 Layout update  (2020-09-12)
-
-  * Update to **new and modern layout**
-  * Support for Chrome improved
-  * Improved robustness: improved error handling in auto flow reduces spontaneous reboots
-  * File server: Option for "DELETE ALL"
-  * WLan: support of spaces in SSID and password
-  * Reference Image: Option for mirror image, option for image update on the fly
-  * additional parameter in `wasserzaehler.html?noerror=true`  to suppress an potential error message
-  * bug fixing
-
-
-
-##### 1.1.3 (2020-09-09)
-
-* **Bug in configuration of analog ROIs corrected** - correction in v.1.0.2 did not work properly
-* Improved update page for the web server (`/html` can be updated via a zip-file, which is provided in `/firmware/html.zip`)
-* Improved Chrome support
-
-##### 1.1.0 (2020-09-06)
-
-* Implementation of "delete complete directory"
-  **Attention: beside the `firmware.bin`, also the content of `/html` needs to be updated!**
-
-
-
-##### 1.0.2 (2020-09-06)
-
-* Bug in configuration of analog ROIs corrected
-* minor bug correction
-
-##### 1.0.1 (2020-09-05)
-
-* preValue.ini Bug corrected
-* minor bug correction
-
-##### 1.0.0 (2020-09-04)
-
-* **First usable version** - compatible to previous project (https://github.com/jomjol/water-meter-system-complete)
-* NEW: 
-  * no docker container for CNN calculation necessary
-  * web based configuration editor on board
-
-##### 0.1.0 (2020-08-07)
-
-* Initial Version
+##### 1.1.3 Initial Version - (2020-09-09)
 
 
 #### [Full Changelog](Changelog.md)
 
-
-
-## Solved topics
-
-* n.a.

code/.helper/copy-esp-idf.bat

@@ -0,0 +1,3 @@
+copy "..\..\code\build\esp32cam-server-only.bin" "..\..\firmware\firmware.bin"
+copy "..\..\code\build\bootloader\bootloader.bin" "..\..\firmware\bootloader.bin"
+copy "..\..\code\build\partition_table\partition-table.bin" "..\..\firmware\partitions.bin"

code/.helper/copy.bat

@@ -1,3 +1,3 @@
-copy "C:\Users\Muell\Documents\Programmieren\GitHub\AI-on-the-edge-device\code\.pio\build\esp32cam\firmware.bin" "C:\Users\Muell\Documents\Programmieren\GitHub\AI-on-the-edge-device\firmware\firmware.bin"
-copy "C:\Users\Muell\Documents\Programmieren\GitHub\AI-on-the-edge-device\code\.pio\build\esp32cam\bootloader.bin" "C:\Users\Muell\Documents\Programmieren\GitHub\AI-on-the-edge-device\firmware\bootloader.bin"
-copy "C:\Users\Muell\Documents\Programmieren\GitHub\AI-on-the-edge-device\code\.pio\build\esp32cam\partitions.bin" "C:\Users\Muell\Documents\Programmieren\GitHub\AI-on-the-edge-device\firmware\partitions.bin"
+copy "..\..\code\.pio\build\esp32cam\firmware.bin" "..\..\firmware\firmware.bin"
+copy "..\..\code\.pio\build\esp32cam\bootloader.bin" "..\..\firmware\bootloader.bin"
+copy "..\..\code\.pio\build\esp32cam\partitions.bin" "..\..\firmware\partitions.bin"

code/.helper/makezip.bat

@@ -1 +1 @@
-powershell Compress-Archive "C:\Users\Muell\Documents\Programmieren\GitHub\AI-on-the-edge-device\sd-card\html\*.*" "C:\Users\Muell\Documents\Programmieren\GitHub\AI-on-the-edge-device\firmware\html.zip"
+powershell Compress-Archive "..\..\sd-card\html\*.*" "..\..\firmware\html.zip"

code/components/connect_wlan/connect_wlan._cpp_

@@ -1,539 +0,0 @@
-#include "connect_wlan.h"
-
-#include <string.h>
-#include "freertos/FreeRTOS.h"
-#include "freertos/task.h"
-#include "freertos/event_groups.h"
-#include "esp_wifi.h"
-#include "esp_log.h"
-
-#include <fstream>
-#include <vector>
-#include <sstream>
-
-#include "Helper.h"
-
-static const char *TAG = "connect_wlan";
-
-std::string ssid = "";
-std::string passphrase = "";
-std::string hostname = "";
-std::string ipaddress = "";
-std::string gw = "";
-std::string netmask = "";
-std::string dns = "";
-
-std::string std_hostname = "watermeter";
-
-#define BLINK_GPIO GPIO_NUM_33
-
-static EventGroupHandle_t s_wifi_event_group;
-
-#define WIFI_CONNECTED_BIT BIT0
-#define WIFI_FAIL_BIT      BIT1
-static int s_retry_num = 0;
-
-
-std::vector<string> ZerlegeZeile(std::string input, std::string _delimiter = "")
-{
-	std::vector<string> Output;
-	std::string delimiter = " =,";
-    if (_delimiter.length() > 0){
-        delimiter = _delimiter;
-    }
-
-	input = trim(input, delimiter);
-	size_t pos = findDelimiterPos(input, delimiter);
-	std::string token;
-	while (pos != std::string::npos) {
-		token = input.substr(0, pos);
-		token = trim(token, delimiter);
-		Output.push_back(token);
-		input.erase(0, pos + 1);
-		input = trim(input, delimiter);
-		pos = findDelimiterPos(input, delimiter);
-	}
-	Output.push_back(input);
-
-	return Output;
-}
-
-
-void blinkstatus(int dauer, int _anzahl)
-{
-    gpio_reset_pin(BLINK_GPIO);
-    gpio_set_direction(BLINK_GPIO, GPIO_MODE_OUTPUT);
-    for (int i = 0; i < _anzahl; ++i)
-    {
-        gpio_set_level(BLINK_GPIO, 0);
-        vTaskDelay(dauer / portTICK_PERIOD_MS);
-        gpio_set_level(BLINK_GPIO, 1);
-        vTaskDelay(dauer / portTICK_PERIOD_MS);          
-    }
-}
-
-
-
-
-void strinttoip4(std::string ip, int &a, int &b, int &c, int &d) {
-    std::stringstream s(ip);
-    char ch; //to temporarily store the '.'
-    s >> a >> ch >> b >> ch >> c >> ch >> d;
-}
-
-
-
-
-static void event_handler_neu(void* arg, esp_event_base_t event_base,
-                                int32_t event_id, void* event_data)
-{
-    if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_STA_START) {
-        blinkstatus(200, 1);
-        esp_wifi_connect();
-    } else if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_STA_DISCONNECTED) {
-        blinkstatus(200, 5);
-        esp_wifi_connect();
-        s_retry_num++;
-        ESP_LOGI(TAG, "retry to connect to the AP");
-    } else if (event_base == IP_EVENT && event_id == IP_EVENT_STA_GOT_IP) {
-        blinkstatus(1000, 3);
-        ip_event_got_ip_t* event = (ip_event_got_ip_t*) event_data;
-        ESP_LOGI(TAG, "got ip:" IPSTR, IP2STR(&event->ip_info.ip));
-        s_retry_num = 0;
-        xEventGroupSetBits(s_wifi_event_group, WIFI_CONNECTED_BIT);
-    }
-}
-
-
-void initialise_wifi()
-{
-    s_wifi_event_group = xEventGroupCreate();
-    ESP_ERROR_CHECK(esp_netif_init());
-    ESP_ERROR_CHECK(esp_event_loop_create_default());
-    esp_netif_create_default_wifi_sta();
-
-    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
-    ESP_ERROR_CHECK(esp_wifi_init(&cfg));
-
-    esp_event_handler_instance_t instance_any_id;
-    esp_event_handler_instance_t instance_got_ip;
-    ESP_ERROR_CHECK(esp_event_handler_instance_register(WIFI_EVENT,
-                                                        ESP_EVENT_ANY_ID,
-                                                        &event_handler_neu,
-                                                        NULL,
-                                                        &instance_any_id));
-    ESP_ERROR_CHECK(esp_event_handler_instance_register(IP_EVENT,
-                                                        IP_EVENT_STA_GOT_IP,
-                                                        &event_handler_neu,
-                                                        NULL,
-                                                        &instance_got_ip));
-
-
-    wifi_config_t wifi_config = { };
-    strcpy((char*)wifi_config.sta.ssid, (const char*)ssid.c_str());
-    strcpy((char*)wifi_config.sta.password, (const char*)passphrase.c_str());
-
-    ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA) );
-    ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config) );
-    ESP_ERROR_CHECK(esp_wifi_start() );
-
-    ESP_LOGI(TAG, "wifi_init_sta finished.");
-
-    // Waiting until either the connection is established (WIFI_CONNECTED_BIT) or connection failed for the maximum
-    // number of re-tries (WIFI_FAIL_BIT). The bits are set by event_handler() (see above) 
-    EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group,
-            WIFI_CONNECTED_BIT | WIFI_FAIL_BIT,
-            pdFALSE,
-            pdFALSE,
-            portMAX_DELAY);
-
-    // xEventGroupWaitBits() returns the bits before the call returned, hence we can test which event actually
-    // happened. 
-    if (bits & WIFI_CONNECTED_BIT) {
-        ESP_LOGI(TAG, "connected to ap SSID:%s password:%s",
-                 ssid.c_str(), passphrase.c_str());
-    } else if (bits & WIFI_FAIL_BIT) {
-        ESP_LOGI(TAG, "Failed to connect to SSID:%s, password:%s",
-                 ssid.c_str(), passphrase.c_str());
-    } else {
-        ESP_LOGE(TAG, "UNEXPECTED EVENT");
-    }
-
-    // The event will not be processed after unregister 
-    ESP_ERROR_CHECK(esp_event_handler_instance_unregister(IP_EVENT, IP_EVENT_STA_GOT_IP, instance_got_ip));
-    ESP_ERROR_CHECK(esp_event_handler_instance_unregister(WIFI_EVENT, ESP_EVENT_ANY_ID, instance_any_id));
-    vEventGroupDelete(s_wifi_event_group);
-
-    tcpip_adapter_ip_info_t ip_info;
-    ESP_ERROR_CHECK(tcpip_adapter_get_ip_info(TCPIP_ADAPTER_IF_STA, &ip_info));
-    ipaddress = std::string(ip4addr_ntoa(&ip_info.ip));
-    netmask = std::string(ip4addr_ntoa(&ip_info.netmask));
-    gw = std::string(ip4addr_ntoa(&ip_info.gw));
-    printf("IPv4 :  %s\n", ip4addr_ntoa(&ip_info.ip));
-    printf("HostName :  %s\n", hostname.c_str());
-}
-
-
-void initialise_wifi_fixed_ip2()
-{
-    s_wifi_event_group = xEventGroupCreate();
-    ESP_ERROR_CHECK(esp_netif_init());
-    ESP_ERROR_CHECK(esp_event_loop_create_default());
-    esp_netif_t *my_sta = esp_netif_create_default_wifi_sta();
-
-    esp_netif_dhcpc_stop(my_sta);
-
-    esp_netif_ip_info_t ip_info;
-
-    int a, b, c, d;
-
-    strinttoip4(ipaddress, a, b, c, d);
-    IP4_ADDR(&ip_info.ip, a, b, c, d);
-
-    strinttoip4(gw, a, b, c, d);
-   	IP4_ADDR(&ip_info.gw, a, b, c, d);
-
-    strinttoip4(netmask, a, b, c, d);
-   	IP4_ADDR(&ip_info.netmask, a, b, c, d);
-
-    esp_netif_set_ip_info(my_sta, &ip_info);
-
-
-    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
-    ESP_ERROR_CHECK(esp_wifi_init(&cfg));
-
-    if (dns.length() > 0) {
-        esp_netif_dns_info_t dns_info;
-        ip4_addr_t ip;
-        ip.addr = esp_ip4addr_aton(dns.c_str());
-        ip_addr_set_ip4_u32(&dns_info.ip, ip.addr);
-        ESP_ERROR_CHECK(esp_netif_set_dns_info(my_sta, ESP_NETIF_DNS_MAIN, &dns_info));
-    }
-
-
-    esp_event_handler_instance_t instance_any_id;
-    esp_event_handler_instance_t instance_got_ip;
-    ESP_ERROR_CHECK(esp_event_handler_instance_register(WIFI_EVENT,
-                                                        ESP_EVENT_ANY_ID,
-                                                        &event_handler_neu,
-                                                        NULL,
-                                                        &instance_any_id));
-    ESP_ERROR_CHECK(esp_event_handler_instance_register(IP_EVENT,
-                                                        IP_EVENT_STA_GOT_IP,
-                                                        &event_handler_neu,
-                                                        NULL,
-                                                        &instance_got_ip));
-
-
-    wifi_config_t wifi_config = { };
-    strcpy((char*)wifi_config.sta.ssid, (const char*)ssid.c_str());
-    strcpy((char*)wifi_config.sta.password, (const char*)passphrase.c_str());
-
-    ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA) );
-    ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config) );
-    ESP_ERROR_CHECK(esp_wifi_start() );
-
-    ESP_LOGI(TAG, "wifi_init_sta finished.");
-
-    // Waiting until either the connection is established (WIFI_CONNECTED_BIT) or connection failed for the maximum
-    // number of re-tries (WIFI_FAIL_BIT). The bits are set by event_handler() (see above) 
-    EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group,
-            WIFI_CONNECTED_BIT | WIFI_FAIL_BIT,
-            pdFALSE,
-            pdFALSE,
-            portMAX_DELAY);
-
-    // xEventGroupWaitBits() returns the bits before the call returned, hence we can test which event actually
-    // happened. 
-    if (bits & WIFI_CONNECTED_BIT) {
-        ESP_LOGI(TAG, "connected to ap SSID:%s password:%s",
-                 ssid.c_str(), passphrase.c_str());
-    } else if (bits & WIFI_FAIL_BIT) {
-        ESP_LOGI(TAG, "Failed to connect to SSID:%s, password:%s",
-                 ssid.c_str(), passphrase.c_str());
-    } else {
-        ESP_LOGE(TAG, "UNEXPECTED EVENT");
-    }
-
-    // The event will not be processed after unregister 
-    ESP_ERROR_CHECK(esp_event_handler_instance_unregister(IP_EVENT, IP_EVENT_STA_GOT_IP, instance_got_ip));
-    ESP_ERROR_CHECK(esp_event_handler_instance_unregister(WIFI_EVENT, ESP_EVENT_ANY_ID, instance_any_id));
-    vEventGroupDelete(s_wifi_event_group);
-
-    tcpip_adapter_ip_info_t ip_info2;
-    ESP_ERROR_CHECK(tcpip_adapter_get_ip_info(TCPIP_ADAPTER_IF_STA, &ip_info2));
-    ipaddress = std::string(ip4addr_ntoa(&ip_info2.ip));
-    netmask = std::string(ip4addr_ntoa(&ip_info2.netmask));
-    gw = std::string(ip4addr_ntoa(&ip_info2.gw));
-}
-
-void ConnectToWLAN()
-{
-    if (ipaddress.length() == 0 || gw.length() == 0 || netmask.length() == 0)
-    {
-        printf("Connect to WLAN with dyn. IP\n");
-        initialise_wifi();    
-    }
-    else
-    {
-        printf("Connect to WLAN with fixed IP\n");
-        initialise_wifi_fixed_ip2();
-    }
-}
-
-
-bool ChangeHostName(std::string fn, std::string _newhostname)
-{
-    if (_newhostname == hostname)
-        return false;
-
-    string line = "";
-    std::vector<string> zerlegt;
-
-    bool found = false;
-
-    std::vector<string> neuesfile;
-
-    FILE* pFile;
-    fn = FormatFileName(fn);
-    pFile = OpenFileAndWait(fn.c_str(), "r");
-
-    printf("file loaded\n");
-
-    if (pFile == NULL)
-        return false;
-
-    char zw[1024];
-    fgets(zw, 1024, pFile);
-    line = std::string(zw);
-
-    while ((line.size() > 0) || !(feof(pFile)))
-    {
-        printf("%s", line.c_str());
-        zerlegt = ZerlegeZeile(line, "=");
-        zerlegt[0] = trim(zerlegt[0], " ");
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "HOSTNAME")){
-            line = "hostname = \"" + _newhostname + "\"\n";
-            found = true;
-        }
-
-        neuesfile.push_back(line);
-
-        if (fgets(zw, 1024, pFile) == NULL)
-        {
-            line = "";
-        }
-        else
-        {
-            line = std::string(zw);
-        }
-    }
-
-    if (!found)
-    {
-        line = "hostname = \"" + _newhostname + "\"\n";
-        neuesfile.push_back(line);        
-    }
-
-    fclose(pFile);
-
-    pFile = OpenFileAndWait(fn.c_str(), "w+");
-
-    for (int i = 0; i < neuesfile.size(); ++i)
-    {
-        fputs(neuesfile[i].c_str(), pFile);
-    }
-
-    fclose(pFile);
-
-    return true;
-}
-
-
-void LoadWlanFromFile(std::string fn)
-{
-    string line = "";
-    std::vector<string> zerlegt;
-    hostname = std_hostname;
-
-    FILE* pFile;
-    fn = FormatFileName(fn);
-
-    pFile = OpenFileAndWait(fn.c_str(), "r");
-    printf("file loaded\n");
-
-    if (pFile == NULL)
-        return;
-
-    char zw[1024];
-    fgets(zw, 1024, pFile);
-    line = std::string(zw);
-
-    while ((line.size() > 0) || !(feof(pFile)))
-    {
-        printf("%s", line.c_str());
-        zerlegt = ZerlegeZeile(line, "=");
-        zerlegt[0] = trim(zerlegt[0], " ");
-        for (int i = 2; i < zerlegt.size(); ++i)
-            zerlegt[i] = zerlegt[i-1] + zerlegt[i];
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "HOSTNAME")){
-            hostname = trim(zerlegt[1]);
-            if ((hostname[0] == '"') && (hostname[hostname.length()-1] == '"')){
-                hostname = hostname.substr(1, hostname.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "SSID")){
-            ssid = trim(zerlegt[1]);
-            if ((ssid[0] == '"') && (ssid[ssid.length()-1] == '"')){
-                ssid = ssid.substr(1, ssid.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "PASSWORD")){
-            passphrase = zerlegt[1];
-            if ((passphrase[0] == '"') && (passphrase[passphrase.length()-1] == '"')){
-                passphrase = passphrase.substr(1, passphrase.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "IP")){
-            ipaddress = zerlegt[1];
-            if ((ipaddress[0] == '"') && (ipaddress[ipaddress.length()-1] == '"')){
-                ipaddress = ipaddress.substr(1, ipaddress.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "GATEWAY")){
-            gw = zerlegt[1];
-            if ((gw[0] == '"') && (gw[gw.length()-1] == '"')){
-                gw = gw.substr(1, gw.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "NETMASK")){
-            netmask = zerlegt[1];
-            if ((netmask[0] == '"') && (netmask[netmask.length()-1] == '"')){
-                netmask = netmask.substr(1, netmask.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "DNS")){
-            dns = zerlegt[1];
-            if ((dns[0] == '"') && (dns[dns.length()-1] == '"')){
-                dns = dns.substr(1, dns.length()-2);
-            }
-        }
-
-
-        if (fgets(zw, 1024, pFile) == NULL)
-        {
-            line = "";
-        }
-        else
-        {
-            line = std::string(zw);
-        }
-    }
-
-    fclose(pFile);
-
-    // Check if Hostname was empty in .ini if yes set to std_hostname
-    if(hostname.length() <= 0){
-        hostname = std_hostname;
-    }
-
-    printf("\nWLan: %s, %s\n", ssid.c_str(), passphrase.c_str());
-    printf("Hostename: %s\n", hostname.c_str());
-    printf("Fixed IP: %s, Gateway %s, Netmask %s, DNS %s\n", ipaddress.c_str(), gw.c_str(), netmask.c_str(), dns.c_str());
-
-}
-
-void LoadNetConfigFromFile(std::string _fn, std::string &_ip, std::string &_gw, std::string &_netmask, std::string &_dns)
-{
-    string line = "";
-    std::vector<string> zerlegt;
-
-
-
-    FILE* pFile;
-    _fn = FormatFileName(_fn);
-    pFile = OpenFileAndWait(_fn.c_str(), "r");
-
-    if (pFile == NULL)
-        return;
-
-    char zw[1024];
-    fgets(zw, 1024, pFile);
-    line = std::string(zw);
-
-    while ((line.size() > 0) || !(feof(pFile)))
-    {
-        printf("%s", line.c_str());
-        zerlegt = ZerlegeZeile(line, "=");
-        zerlegt[0] = trim(zerlegt[0], " ");
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "IP")){
-            _ip = zerlegt[1];
-            if ((_ip[0] == '"') && (_ip[_ip.length()-1] == '"')){
-                _ip = _ip.substr(1, _ip.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "GATEWAY")){
-            _gw = zerlegt[1];
-            if ((_gw[0] == '"') && (_gw[_gw.length()-1] == '"')){
-                _gw = _gw.substr(1, _gw.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "NETMASK")){
-            _netmask = zerlegt[1];
-            if ((_netmask[0] == '"') && (_netmask[_netmask.length()-1] == '"')){
-                _netmask = _netmask.substr(1, _netmask.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "DNS")){
-            _dns = zerlegt[1];
-            if ((_dns[0] == '"') && (_dns[_dns.length()-1] == '"')){
-                _dns = _dns.substr(1, _dns.length()-2);
-            }
-        }
-
-        if (fgets(zw, 1024, pFile) == NULL)
-        {
-            line = "";
-        }
-        else
-        {
-            line = std::string(zw);
-        }
-    }
-
-    fclose(pFile);
-}
-
-
-std::string getHostname(){
-    return hostname;
-}
-
-std::string getIPAddress(){
-    return ipaddress;
-}
-
-std::string getSSID(){
-    return ssid;
-}
-
-std::string getNetMask(){
-    return netmask;
-}
-
-std::string getGW(){
-    return gw;
-}
-

code/components/connect_wlan/connect_wlan._h_

@@ -1,21 +0,0 @@
-//#ifndef CONNECT_WLAN_H
-//#define CONNECT_WLAN_H
-
-#include <string>
-#include "driver/gpio.h"
-
-const int CONNECTED_BIT = BIT0;
-
-void ConnectToWLAN();
-
-void LoadWlanFromFile(std::string fn);
-
-bool ChangeHostName(std::string fn, std::string _newhostname);
-
-std::string getHostname();
-std::string getIPAddress();
-std::string getSSID();
-std::string getNetMask();
-std::string getGW();
-
-//#endif

code/components/connect_wlan/connect_wlan.cpp

@@ -1,489 +0,0 @@
-#include "connect_wlan.h"
-
-
-
-#include <string.h>
-#include "esp_wifi.h"
-#include "esp_event_loop.h"
-#include "freertos/event_groups.h"
-#include "esp_log.h"
-
-#include <fstream>
-#include <string>
-#include <vector>
-#include <sstream>
-#include <iostream>
-
-#include <arpa/inet.h>
-
-#include "Helper.h"
-
-
-
-
-static const char *MAIN_TAG = "connect_wlan";
-
-std::string ssid = "";
-std::string passphrase = "";
-std::string hostname = "";
-std::string ipaddress = "";
-std::string gw = "";
-std::string netmask = "";
-std::string dns = "";
-std::string std_hostname = "watermeter";
-
-static EventGroupHandle_t wifi_event_group;
-
-
-#define BLINK_GPIO GPIO_NUM_33
-
-
-std::vector<string> ZerlegeZeile(std::string input, std::string _delimiter = "")
-{
-	std::vector<string> Output;
-	std::string delimiter = " =,";
-    if (_delimiter.length() > 0){
-        delimiter = _delimiter;
-    }
-
-	input = trim(input, delimiter);
-	size_t pos = findDelimiterPos(input, delimiter);
-	std::string token;
-	while (pos != std::string::npos) {
-		token = input.substr(0, pos);
-		token = trim(token, delimiter);
-		Output.push_back(token);
-		input.erase(0, pos + 1);
-		input = trim(input, delimiter);
-		pos = findDelimiterPos(input, delimiter);
-	}
-	Output.push_back(input);
-
-	return Output;
-}
-
-
-
-
-void wifi_connect(){
-    wifi_config_t cfg = { };
-    strcpy((char*)cfg.sta.ssid, (const char*)ssid.c_str());
-    strcpy((char*)cfg.sta.password, (const char*)passphrase.c_str());
-    
-    ESP_ERROR_CHECK( esp_wifi_disconnect() );
-    ESP_ERROR_CHECK( esp_wifi_set_config(ESP_IF_WIFI_STA, &cfg) );
-    ESP_ERROR_CHECK( esp_wifi_connect() );
-}
-
-
-
-void blinkstatus(int dauer, int _anzahl)
-{
-    gpio_reset_pin(BLINK_GPIO);
-    gpio_set_direction(BLINK_GPIO, GPIO_MODE_OUTPUT);
-    for (int i = 0; i < _anzahl; ++i)
-    {
-        gpio_set_level(BLINK_GPIO, 0);
-        vTaskDelay(dauer / portTICK_PERIOD_MS);
-        gpio_set_level(BLINK_GPIO, 1);
-        vTaskDelay(dauer / portTICK_PERIOD_MS);          
-    }
-}
-
-static esp_err_t event_handler(void *ctx, system_event_t *event)
-{
-    switch(event->event_id) {
-    case SYSTEM_EVENT_STA_START:
-        blinkstatus(200, 1);
-        wifi_connect();
-        break;
-    case SYSTEM_EVENT_STA_GOT_IP:
-        xEventGroupSetBits(wifi_event_group, CONNECTED_BIT);
-        blinkstatus(1000, 3);
-        break;
-    case SYSTEM_EVENT_STA_DISCONNECTED:
-        blinkstatus(200, 5);
-        esp_wifi_connect();
-        xEventGroupClearBits(wifi_event_group, CONNECTED_BIT);
-        break;
-    default:
-        break;
-    }
-    return ESP_OK;
-}
-
-void initialise_wifi()
-{
-    ESP_ERROR_CHECK(esp_event_loop_init(event_handler, NULL) );
-    wifi_event_group = xEventGroupCreate();  
-
-    esp_log_level_set("wifi", ESP_LOG_NONE); // disable wifi driver logging
-    tcpip_adapter_init();
-    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
-    ESP_ERROR_CHECK( esp_wifi_init(&cfg) );
-    ESP_ERROR_CHECK( esp_wifi_set_mode(WIFI_MODE_STA) );
-    ESP_ERROR_CHECK( esp_wifi_start() );
-    esp_err_t ret = tcpip_adapter_set_hostname(TCPIP_ADAPTER_IF_STA , hostname.c_str());
-    if(ret != ESP_OK ){
-      ESP_LOGE(MAIN_TAG,"failed to set hostname:%d",ret);  
-    }
-    xEventGroupWaitBits(wifi_event_group,CONNECTED_BIT,true,true,portMAX_DELAY);
-    tcpip_adapter_ip_info_t ip_info;
-    ESP_ERROR_CHECK(tcpip_adapter_get_ip_info(TCPIP_ADAPTER_IF_STA, &ip_info));
-    ipaddress = std::string(ip4addr_ntoa(&ip_info.ip));
-    netmask = std::string(ip4addr_ntoa(&ip_info.netmask));
-    gw = std::string(ip4addr_ntoa(&ip_info.gw));
-    printf("IPv4 :  %s\n", ip4addr_ntoa(&ip_info.ip));
-    printf("HostName :  %s\n", hostname.c_str());
-}
-
-///////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////
-
-void strinttoip4(std::string ip, int &a, int &b, int &c, int &d) {
-    std::stringstream s(ip);
-    char ch; //to temporarily store the '.'
-    s >> a >> ch >> b >> ch >> c >> ch >> d;
-}
-
-void initialise_wifi_fixed_ip()
-{
-
-    wifi_event_group = xEventGroupCreate();
-
-    ESP_ERROR_CHECK(esp_netif_init());
-    ESP_ERROR_CHECK(esp_event_loop_create_default());
-
-    esp_netif_t *my_sta = esp_netif_create_default_wifi_sta();
-
-    esp_netif_dhcpc_stop(my_sta);
-
-    esp_netif_ip_info_t ip_info;
-
-    int a, b, c, d;
-
-    strinttoip4(ipaddress, a, b, c, d);
-    IP4_ADDR(&ip_info.ip, a, b, c, d);
-
-    strinttoip4(gw, a, b, c, d);
-   	IP4_ADDR(&ip_info.gw, a, b, c, d);
-
-    strinttoip4(netmask, a, b, c, d);
-   	IP4_ADDR(&ip_info.netmask, a, b, c, d);
-
-    esp_netif_set_ip_info(my_sta, &ip_info);
-
-    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
-    ESP_ERROR_CHECK(esp_wifi_init(&cfg));
-
-    if (dns.length() > 0) {
-        esp_netif_dns_info_t dns_info;
-        ip4_addr_t ip;
-        ip.addr = esp_ip4addr_aton(dns.c_str());
-        ip_addr_set_ip4_u32(&dns_info.ip, ip.addr);
-        ESP_ERROR_CHECK(esp_netif_set_dns_info(my_sta, ESP_NETIF_DNS_MAIN, &dns_info));
-    }
-
-    ESP_ERROR_CHECK(esp_event_loop_init(event_handler, NULL) );
-
-    wifi_config_t wifi_config = { };
-    strcpy((char*)wifi_config.sta.ssid, (const char*)ssid.c_str());
-    strcpy((char*)wifi_config.sta.password, (const char*)passphrase.c_str());
-
-    ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA) );
-    ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config) );
-    ESP_ERROR_CHECK(esp_wifi_start() );
-
-    ESP_LOGI(MAIN_TAG, "wifi_init_sta finished.");
-
-    EventBits_t bits = xEventGroupWaitBits(wifi_event_group,CONNECTED_BIT,true,true,portMAX_DELAY);
-
-    if (bits & CONNECTED_BIT) {
-        ESP_LOGI(MAIN_TAG, "connected to ap SSID:%s password:%s",
-                 ssid.c_str(), passphrase.c_str());
-    } else  {
-        ESP_LOGI(MAIN_TAG, "Failed to connect to SSID:%s, password:%s",
-                 ssid.c_str(), passphrase.c_str());
-    }
-    tcpip_adapter_ip_info_t ip_info2;
-    ESP_ERROR_CHECK(tcpip_adapter_get_ip_info(TCPIP_ADAPTER_IF_STA, &ip_info2));
-    ipaddress = std::string(ip4addr_ntoa(&ip_info2.ip));
-    netmask = std::string(ip4addr_ntoa(&ip_info2.netmask));
-    gw = std::string(ip4addr_ntoa(&ip_info2.gw));
-
-//    vEventGroupDelete(wifi_event_group);
-}
-
-
-
-void ConnectToWLAN()
-{
-    if (ipaddress.length() == 0 || gw.length() == 0 || netmask.length() == 0)
-    {
-        printf("Connect to WLAN with dyn. IP\n");
-        initialise_wifi();    
-    }
-    else
-    {
-        printf("Connect to WLAN with fixed IP\n");
-        initialise_wifi_fixed_ip();
-    }
-}
-
-
-
-bool ChangeHostName(std::string fn, std::string _newhostname)
-{
-    if (_newhostname == hostname)
-        return false;
-
-    string line = "";
-    std::vector<string> zerlegt;
-
-    bool found = false;
-
-    std::vector<string> neuesfile;
-
-    FILE* pFile;
-    fn = FormatFileName(fn);
-    pFile = OpenFileAndWait(fn.c_str(), "r");
-
-    printf("file loaded\n");
-
-    if (pFile == NULL)
-        return false;
-
-    char zw[1024];
-    fgets(zw, 1024, pFile);
-    line = std::string(zw);
-
-    while ((line.size() > 0) || !(feof(pFile)))
-    {
-        printf("%s", line.c_str());
-        zerlegt = ZerlegeZeile(line, "=");
-        zerlegt[0] = trim(zerlegt[0], " ");
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "HOSTNAME")){
-            line = "hostname = \"" + _newhostname + "\"\n";
-            found = true;
-        }
-
-        neuesfile.push_back(line);
-
-        if (fgets(zw, 1024, pFile) == NULL)
-        {
-            line = "";
-        }
-        else
-        {
-            line = std::string(zw);
-        }
-    }
-
-    if (!found)
-    {
-        line = "hostname = \"" + _newhostname + "\"\n";
-        neuesfile.push_back(line);        
-    }
-
-    fclose(pFile);
-
-    pFile = OpenFileAndWait(fn.c_str(), "w+");
-
-    for (int i = 0; i < neuesfile.size(); ++i)
-    {
-        fputs(neuesfile[i].c_str(), pFile);
-    }
-
-    fclose(pFile);
-
-    return true;
-}
-
-
-void LoadWlanFromFile(std::string fn)
-{
-    string line = "";
-    std::vector<string> zerlegt;
-    hostname = std_hostname;
-
-    FILE* pFile;
-    fn = FormatFileName(fn);
-
-    pFile = OpenFileAndWait(fn.c_str(), "r");
-    printf("file loaded\n");
-
-    if (pFile == NULL)
-        return;
-
-    char zw[1024];
-    fgets(zw, 1024, pFile);
-    line = std::string(zw);
-
-    while ((line.size() > 0) || !(feof(pFile)))
-    {
-        printf("%s", line.c_str());
-        zerlegt = ZerlegeZeile(line, "=");
-        zerlegt[0] = trim(zerlegt[0], " ");
-        for (int i = 2; i < zerlegt.size(); ++i)
-            zerlegt[i] = zerlegt[i-1] + zerlegt[i];
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "HOSTNAME")){
-            hostname = trim(zerlegt[1]);
-            if ((hostname[0] == '"') && (hostname[hostname.length()-1] == '"')){
-                hostname = hostname.substr(1, hostname.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "SSID")){
-            ssid = trim(zerlegt[1]);
-            if ((ssid[0] == '"') && (ssid[ssid.length()-1] == '"')){
-                ssid = ssid.substr(1, ssid.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "PASSWORD")){
-            passphrase = zerlegt[1];
-            if ((passphrase[0] == '"') && (passphrase[passphrase.length()-1] == '"')){
-                passphrase = passphrase.substr(1, passphrase.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "IP")){
-            ipaddress = zerlegt[1];
-            if ((ipaddress[0] == '"') && (ipaddress[ipaddress.length()-1] == '"')){
-                ipaddress = ipaddress.substr(1, ipaddress.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "GATEWAY")){
-            gw = zerlegt[1];
-            if ((gw[0] == '"') && (gw[gw.length()-1] == '"')){
-                gw = gw.substr(1, gw.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "NETMASK")){
-            netmask = zerlegt[1];
-            if ((netmask[0] == '"') && (netmask[netmask.length()-1] == '"')){
-                netmask = netmask.substr(1, netmask.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "DNS")){
-            dns = zerlegt[1];
-            if ((dns[0] == '"') && (dns[dns.length()-1] == '"')){
-                dns = dns.substr(1, dns.length()-2);
-            }
-        }
-
-
-        if (fgets(zw, 1024, pFile) == NULL)
-        {
-            line = "";
-        }
-        else
-        {
-            line = std::string(zw);
-        }
-    }
-
-    fclose(pFile);
-
-    // Check if Hostname was empty in .ini if yes set to std_hostname
-    if(hostname.length() <= 0){
-        hostname = std_hostname;
-    }
-
-    printf("\nWLan: %s, %s\n", ssid.c_str(), passphrase.c_str());
-    printf("Hostename: %s\n", hostname.c_str());
-    printf("Fixed IP: %s, Gateway %s, Netmask %s, DNS %s\n", ipaddress.c_str(), gw.c_str(), netmask.c_str(), dns.c_str());
-
-}
-
-void LoadNetConfigFromFile(std::string fn, std::string &_ip, std::string &_gw, std::string &_netmask, std::string &_dns)
-{
-    string line = "";
-    std::vector<string> zerlegt;
-
-    FILE* pFile;
-    fn = FormatFileName(fn);
-    pFile = OpenFileAndWait(fn.c_str(), "r");
-
-    printf("file loaded\n");
-
-    if (pFile == NULL)
-        return;
-
-    char zw[1024];
-    fgets(zw, 1024, pFile);
-    line = std::string(zw);
-
-    while ((line.size() > 0) || !(feof(pFile)))
-    {
-        printf("%s", line.c_str());
-        zerlegt = ZerlegeZeile(line, "=");
-        zerlegt[0] = trim(zerlegt[0], " ");
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "IP")){
-            _ip = zerlegt[1];
-            if ((_ip[0] == '"') && (_ip[_ip.length()-1] == '"')){
-                _ip = _ip.substr(1, _ip.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "GATEWAY")){
-            _gw = zerlegt[1];
-            if ((_gw[0] == '"') && (_gw[_gw.length()-1] == '"')){
-                _gw = _gw.substr(1, _gw.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "NETMASK")){
-            _netmask = zerlegt[1];
-            if ((_netmask[0] == '"') && (_netmask[_netmask.length()-1] == '"')){
-                _netmask = _netmask.substr(1, _netmask.length()-2);
-            }
-        }
-
-        if ((zerlegt.size() > 1) && (toUpper(zerlegt[0]) == "DNS")){
-            _dns = zerlegt[1];
-            if ((_dns[0] == '"') && (_dns[_dns.length()-1] == '"')){
-                _dns = _dns.substr(1, _dns.length()-2);
-            }
-        }
-
-        if (fgets(zw, 1024, pFile) == NULL)
-        {
-            line = "";
-        }
-        else
-        {
-            line = std::string(zw);
-        }
-    }
-
-    fclose(pFile);
-}
-
-
-std::string getHostname(){
-    return hostname;
-}
-
-std::string getIPAddress(){
-    return ipaddress;
-}
-
-std::string getSSID(){
-    return ssid;
-}
-
-std::string getNetMask(){
-    return netmask;
-}
-
-std::string getGW(){
-    return gw;
-}
-

code/components/connect_wlan/connect_wlan.h

@@ -1,20 +0,0 @@
-#ifndef CONNECT_WLAN_H
-#define CONNECT_WLAN_H
-
-#include <string>
-#include "driver/gpio.h"
-
-const int CONNECTED_BIT = BIT0;
-void ConnectToWLAN();
-
-void LoadWlanFromFile(std::string fn);
-
-bool ChangeHostName(std::string fn, std::string _newhostname);
-
-std::string getHostname();
-std::string getIPAddress();
-std::string getSSID();
-std::string getNetMask();
-std::string getGW();
-
-#endif

code/components/esp-nn/.gitignore

@@ -0,0 +1,57 @@
+.config
+*.o
+*.i
+*.s
+*.orig
+*.pyc
+
+# gtags
+GTAGS
+GRTAGS
+GPATH
+
+# emacs
+.dir-locals.el
+
+# emacs temp file suffixes
+*~
+.#*
+\#*#
+
+# eclipse setting
+.settings
+
+# MacOS directory files
+.DS_Store
+
+# Example project files
+examples/**/sdkconfig
+examples/**/sdkconfig.old
+examples/**/build
+
+# Test app files
+test_app/build
+test_app/sdkconfig
+test_app/sdkconfig.old
+
+# Doc build artifacts
+docs/_build/
+docs/doxygen-warning-log.txt
+docs/sphinx-warning-log.txt
+docs/sphinx-warning-log-sanitized.txt
+docs/xml/
+docs/xml_in/
+docs/man/
+docs/doxygen_sqlite3.db
+
+TEST_LOGS
+
+
+# gcov coverage reports
+*.gcda
+*.gcno
+coverage.info
+coverage_report/
+
+# VS Code Settings
+.vscode/

code/components/esp-nn/.gitlab-ci.yml

@@ -0,0 +1,55 @@
+stages:
+  - build
+
+variables:
+  BATCH_BUILD: "1"
+  V: "0"
+  MAKEFLAGS: "-j8 --no-keep-going"
+  IDF_PATH: "$CI_PROJECT_DIR/esp-idf"
+  LOG_PATH: "$CI_PROJECT_DIR"
+
+.set_git_config: &set_git_config
+  # Set git config
+  - git config user.email "test@espressif.com"
+  - git config user.name "Espressif"
+
+.add_ssh_key: &add_ssh_key
+  # Add gitlab ssh key
+  - mkdir -p ~/.ssh
+  - chmod 700 ~/.ssh
+  - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
+  - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+  - chmod 600 ~/.ssh/id_rsa
+  - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+
+before_script:
+  # Add gitlab ssh key
+  - *add_ssh_key
+  # Set git config
+  - *set_git_config
+
+.build_esp32s3: &build_esp32s3
+  - idf.py set-target esp32s3 build
+
+.build_esp32: &build_esp32
+  - idf.py set-target esp32 build
+
+build_demo:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env:esp-nn
+  tags:
+    - build
+  script:
+    # Clone IDF
+    - git clone --recursive --single-branch -b release/v4.4 --reference-if-able /local_references/gitlab/ https://gitlab-ci-token:${BOT_TOKEN}@gitlab.espressif.cn:6688/espressif/esp-idf.git
+    - cd esp-idf
+    - ./install.sh
+    - . ./export.sh
+    - cd ..
+    # Build examples now
+    - cd test_app
+    # Build esp32s3
+    - *build_esp32s3
+    # Build esp32
+    - *build_esp32
+    - cd -

code/components/esp-nn/CMakeLists.txt

@@ -0,0 +1,50 @@
+idf_build_get_property(idf_target IDF_TARGET)
+
+set(c_srcs
+    "src/activation_functions/esp_nn_relu_ansi.c"
+    "src/basic_math/esp_nn_add_ansi.c"
+    "src/basic_math/esp_nn_mul_ansi.c"
+    "src/convolution/esp_nn_conv_ansi.c"
+    "src/convolution/esp_nn_conv_opt.c"
+    "src/convolution/esp_nn_depthwise_conv_ansi.c"
+    "src/convolution/esp_nn_depthwise_conv_opt.c"
+    "src/fully_connected/esp_nn_fully_connected_ansi.c"
+    "src/softmax/esp_nn_softmax_ansi.c"
+    "src/softmax/esp_nn_softmax_opt.c"
+    "src/pooling/esp_nn_avg_pool_ansi.c"
+    "src/pooling/esp_nn_max_pool_ansi.c")
+
+if(CONFIG_IDF_TARGET_ESP32S3)
+    set(s3_srcs
+        "src/common/esp_nn_common_functions_esp32s3.S"
+        "src/common/esp_nn_multiply_by_quantized_mult_esp32s3.S"
+        "src/common/esp_nn_multiply_by_quantized_mult_ver1_esp32s3.S"
+        "src/activation_functions/esp_nn_relu_s8_esp32s3.S"
+        "src/basic_math/esp_nn_add_s8_esp32s3.S"
+        "src/basic_math/esp_nn_mul_s8_esp32s3.S"
+        "src/convolution/esp_nn_conv_esp32s3.c"
+        "src/convolution/esp_nn_depthwise_conv_s8_esp32s3.c"
+        "src/convolution/esp_nn_conv_s16_mult8_esp32s3.S"
+        "src/convolution/esp_nn_conv_s8_mult8_1x1_esp32s3.S"
+        "src/convolution/esp_nn_conv_s16_mult4_1x1_esp32s3.S"
+        "src/convolution/esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3.S"
+        "src/convolution/esp_nn_depthwise_conv_s16_mult1_esp32s3.S"
+        "src/convolution/esp_nn_depthwise_conv_s16_mult1_3x3_esp32s3.S"
+        "src/convolution/esp_nn_depthwise_conv_s16_mult1_3x3_no_pad_esp32s3.S"
+        "src/convolution/esp_nn_depthwise_conv_s16_mult8_3x3_esp32s3.S"
+        "src/convolution/esp_nn_depthwise_conv_s16_mult4_esp32s3.S"
+        "src/convolution/esp_nn_depthwise_conv_s16_mult8_esp32s3.S"
+        "src/fully_connected/esp_nn_fully_connected_s8_esp32s3.S"
+        "src/pooling/esp_nn_max_pool_s8_esp32s3.S"
+        "src/pooling/esp_nn_avg_pool_s8_esp32s3.S")
+endif()
+
+idf_component_register(SRCS "${c_srcs}"
+                            "${s3_srcs}"
+                       INCLUDE_DIRS "include" "src/common")
+
+if(CONFIG_IDF_TARGET_ESP32S3)
+    target_compile_options(${COMPONENT_LIB} PRIVATE -mlongcalls -fno-unroll-loops -O2 -Wno-unused-function)
+else()
+    target_compile_options(${COMPONENT_LIB} PRIVATE -Wno-unused-function)
+endif()

code/components/esp-nn/Kconfig.projbuild

@@ -0,0 +1,29 @@
+menu "ESP-NN"
+
+choice NN_OPTIMIZATIONS
+   bool "Optimization for nn functions"
+   default NN_OPTIMIZED
+   help
+      Use ANSI-C versions for verification and debug purpose.
+      Optimisations are automatically picked up for a chipset.
+      For ESP32-S3, assembly optimisations are selected.
+      For other platforms(viz., ESP32, ESP32-C3), generic optimisations are used.
+
+config NN_ANSI_C
+   bool "ANSI C"
+   help
+      ANSI C versions for verification and debug purposes.
+config NN_OPTIMIZED
+   bool "Optimized versions"
+   help
+      Optimisations are automatically picked up for a chipset.
+      For ESP32-S3, assembly optimisations are selected.
+      For other platforms(viz., ESP32, ESP32-C3), generic optimisations are used.
+endchoice
+
+config NN_OPTIMIZATIONS
+   int
+   default 0 if NN_ANSI_C
+   default 1 if NN_OPTIMIZED
+
+endmenu

code/components/esp-nn/README.md

@@ -0,0 +1,55 @@
+# ESP-NN
+
+The library contains optimised NN (Neural Network) functions for various Espressif chipsets.
+
+* Supported platforms:
+   * TensorFlow Lite Micro (TFLite Micro). Repo can be found [here](https://github.com/espressif/tflite-micro-esp-examples)
+
+* Supported ESP chipsets include:
+   * ESP32-S3 (Assembly versions optimised to benefit from vector instructions of ESP32-S3)
+   * ESP32 (Generic optimisations)
+   * ESP32-C3 (Generic optimisations)
+
+## Performance
+
+### Kernelwise performance for s8 versions:
+
+  * Kernelwise performance on ESP32-S3 chip
+    * Numbers are ticks taken for kernel to execute
+    * Chip config: 240MHz, SPI: QPI 80MHz, Data cache: 64KB
+
+    | Function        | ANSI C  | ESP32-S3 Opt  | Opt Ratio | Data info   | Memory    |
+    | ----------------| --------|---------|---------|-------------|-----------|
+    | elementwise_add | 320397  | 87119   | 3.68    | size = 1615 | External  |
+    | elementwise_mul | 125958  | 44239   | 2.85    | size = 1615 | External  |
+    | convolution     | 4663012 | 428675  | 10.88   | input(10,10), filter(64x1x1x64) | External |
+    | convolution     | 301014  | 32433   | 9.28    | input(8,8), filter(16x1x1x16) | External |
+    | convolution     | 2115418 | 1020923 | 2.07    | input(10,10), filter(64x3x3x3) | External |
+    | depthwise conv  | 1190062 | 203278  | 5.85    | input (18, 18), pad(0,0), stride(1,1) filter: 1x3x3x16 | External |
+    | depthwise conv  | 837072  | 182335  | 4.59    | input (12, 12), pad(1,1), stride(1,1)  filter: 8x5x5x4 | External |
+    | max pool        | 485714  | 76747   | 6.33    | input(16,16), filter (1x3x3x16) | Internal |
+    | avg pool        | 541462  | 160580  | 3.37    | input(16,16), filter (1x3x3x16) | Internal |
+    | fully connected | 15853   | 9547    | 1.66    | len: 265, ch = 3 | Internal |
+    | prelu (relu6)   | 19472   | 2734    | 7.12    | size, 1615  | Internal  |
+
+
+## Configuration
+
+  * To configure, please use `idf.py menuconfig` and under `ESP-NN` select `NN_OPTIMIZATIONS`
+  * There are two options presented:
+     * Optimized versions
+     * ANSI C
+
+  * Default selection is for `Optimized versions`. For ESP32-S3, assembly versions are automatically selected, whereas for other chipsets (viz., ESP32, ESP32-C3), generic optimisations are selected.
+  * For debugging purposes, you may want to select `ANSI C` reference versions.
+
+
+## Contributing
+
+If you encounter an issue with ESP-NN, or wish to submit a feature request, please use the Issues section on the Github.
+
+For general questions related to this library, please use the esp32.com forum.
+
+## Copyrights and License
+
+All original source code in this repository is Copyright (C) 2020-2021 Espressif Systems. This source code is licensed under the Apache License 2.0 as described in the file LICENSE.

code/components/esp-nn/include/esp_nn.h

@@ -0,0 +1,46 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#if defined(CONFIG_NN_OPTIMIZED)
+// select apt optimisations
+#ifdef CONFIG_IDF_TARGET_ESP32S3
+#define ARCH_ESP32_S3 1
+#endif
+#ifdef CONFIG_IDF_TARGET_ESP32
+#define ARCH_ESP32 1
+#endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* reference kernels included by default */
+#include "esp_nn_ansi_headers.h"
+
+#if defined(CONFIG_NN_OPTIMIZED)
+#if defined(ARCH_ESP32_S3)
+#include "esp_nn_esp32s3.h"
+#else // for other platforms use generic optimisations
+#include "esp_nn_generic_opt.h"
+#endif // #if defined(ARCH_ESP32_S3)
+#else
+#include "esp_nn_ansi_c.h"
+#endif
+
+#ifdef __cplusplus
+}
+#endif

code/components/esp-nn/include/esp_nn_ansi_c.h

@@ -0,0 +1,47 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file        Header definitions to include for ANSI C versions.
+ *              These are just typedefs to pick up ANSI versions.
+ */
+
+#pragma once
+
+#include "esp_nn_defs.h"
+#include "esp_nn_ansi_headers.h"
+
+#define esp_nn_add_elementwise_s8 esp_nn_add_elementwise_s8_ansi
+#define esp_nn_mul_elementwise_s8 esp_nn_mul_elementwise_s8_ansi
+
+#define esp_nn_depthwise_conv_s8 esp_nn_depthwise_conv_s8_ansi
+
+#define esp_nn_conv_s8 esp_nn_conv_s8_ansi
+
+#define esp_nn_get_conv_scratch_size esp_nn_get_conv_scratch_size_ansi
+#define esp_nn_set_conv_scratch_buf esp_nn_set_conv_scratch_buf_ansi
+
+#define esp_nn_get_depthwise_conv_scratch_size esp_nn_get_depthwise_conv_scratch_size_ansi
+#define esp_nn_set_depthwise_conv_scratch_buf esp_nn_set_depthwise_conv_scratch_buf_ansi
+
+#define esp_nn_relu6_s8 esp_nn_relu6_s8_ansi
+
+#define esp_nn_avg_pool_s8 esp_nn_avg_pool_s8_ansi
+#define esp_nn_max_pool_s8 esp_nn_max_pool_s8_ansi
+
+#define esp_nn_fully_connected_s8 esp_nn_fully_connected_s8_ansi
+
+#define esp_nn_get_softmax_scratch_size esp_nn_get_softmax_scratch_size_ansi
+#define esp_nn_set_softmax_scratch_buf esp_nn_set_softmax_scratch_buf_ansi
+#define esp_nn_softmax_s8 esp_nn_softmax_s8_ansi

code/components/esp-nn/include/esp_nn_ansi_headers.h

@@ -0,0 +1,309 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+/**
+ * @file        Header definitions to include for esp_nn reference functions
+ */
+
+#include "esp_nn_defs.h"
+/************************** Basic math functions ****************************/
+
+/**
+ * @brief       elementwise addition
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ *
+ *              shift values are expected to be <= 0
+ */
+void esp_nn_add_elementwise_s8_ansi(const int8_t *input1_data,
+                                    const int8_t *input2_data,
+                                    const int32_t input1_offset,
+                                    const int32_t input2_offset,
+                                    const int32_t input1_mult,
+                                    const int32_t input2_mult,
+                                    const int32_t input1_shift,
+                                    const int32_t input2_shift,
+                                    const int32_t left_shift,
+                                    int8_t *output,
+                                    const int32_t out_offset,
+                                    const int32_t out_mult,
+                                    const int32_t out_shift,
+                                    const int32_t activation_min,
+                                    const int32_t activation_max,
+                                    const int32_t size);
+/**
+ * @brief       elementwise multiplication
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ *
+ *              output shift is expected to be <= 0
+ */
+void esp_nn_mul_elementwise_s8_ansi(const int8_t *input1_data,
+                                    const int8_t *input2_data,
+                                    const int32_t input1_offset,
+                                    const int32_t input2_offset,
+                                    int8_t *output,
+                                    const int32_t out_offset,
+                                    const int32_t out_mult,
+                                    const int32_t out_shift,
+                                    const int32_t activation_min,
+                                    const int32_t activation_max,
+                                    const int32_t size);
+
+
+/************************** Convolution functions *****************************/
+
+/**
+ * @brief       depthwise convolution per channel
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              Version used in tflite is per channel.
+ *              This version follows the same footsprints.
+ *              Meaning, it has per out_channel shift and multiplier for
+ *              requantization
+ *
+ *              optimization notes: Though input_offset is int32 type,
+ *              offset values are contained in 8 bits [-128, 127]
+ */
+void esp_nn_depthwise_conv_s8_ansi(const data_dims_t *input_dims,
+                                   const int8_t *input_data,
+                                   const data_dims_t *filter_dims,
+                                   const int8_t *filter_data,
+                                   const int32_t *bias,
+                                   const data_dims_t *output_dims,
+                                   int8_t *out_data,
+                                   const dw_conv_params_t *conv_params,
+                                   const quant_data_t *quant_data);
+
+/**
+ * @brief       2d-convolution channelwise
+ *
+ * @note        operation: result += (input + offset) * filter
+ *
+ *              inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ */
+void esp_nn_conv_s8_ansi(const data_dims_t *input_dims,
+                         const int8_t *input_data,
+                         const data_dims_t *filter_dims,
+                         const int8_t *filter_data,
+                         const int32_t *bias,
+                         const data_dims_t *output_dims,
+                         int8_t *out_data,
+                         const conv_params_t *conv_params,
+                         const quant_data_t *quant_data);
+
+int esp_nn_get_conv_scratch_size_ansi(const data_dims_t *input_dims,
+                                      const data_dims_t *filter_dims,
+                                      const data_dims_t *output_dims,
+                                      const conv_params_t *conv_params);
+void esp_nn_set_conv_scratch_buf_ansi(const void *buf);
+
+int esp_nn_get_depthwise_conv_scratch_size_ansi(const data_dims_t *input_dims,
+                                                const data_dims_t *filter_dims,
+                                                const data_dims_t *output_dims,
+                                                const dw_conv_params_t *conv_params);
+void esp_nn_set_depthwise_conv_scratch_buf_ansi(const void *buf);
+
+/************************** Activation functions *****************************/
+
+/**
+ * @brief       relu6
+ *
+ * @note        inout: int8_t
+ */
+void esp_nn_relu6_s8_ansi(int8_t *data, uint16_t size);
+
+/************************** Pooling functions *****************************/
+
+
+/**
+ * @brief       max_pool
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ */
+void esp_nn_max_pool_s8_ansi(const int8_t *input,
+                             const uint16_t input_wd,
+                             const uint16_t input_ht,
+                             int8_t *output,
+                             const uint16_t output_wd,
+                             const uint16_t output_ht,
+                             const uint16_t stride_wd,
+                             const uint16_t stride_ht,
+                             const uint16_t filter_wd,
+                             const uint16_t filter_ht,
+                             const uint16_t pad_wd,
+                             const uint16_t pad_ht,
+                             const int32_t activation_min,
+                             const int32_t activation_max,
+                             const uint16_t channels);
+
+/**
+ * @brief       avg_pool
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ */
+void esp_nn_avg_pool_s8_ansi(const int8_t *input,
+                             const uint16_t input_wd,
+                             const uint16_t input_ht,
+                             int8_t *output,
+                             const uint16_t output_wd,
+                             const uint16_t output_ht,
+                             const uint16_t stride_wd,
+                             const uint16_t stride_ht,
+                             const uint16_t filter_wd,
+                             const uint16_t filter_ht,
+                             const uint16_t pad_wd,
+                             const uint16_t pad_ht,
+                             const int32_t activation_min,
+                             const int32_t activation_max,
+                             const uint16_t channels);
+
+
+/************************** Fully connected functions ***********************/
+
+/**
+ * @brief       fully connected
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ */
+void esp_nn_fully_connected_s8_ansi(const int8_t *input_data,
+                                    const int32_t input_offset,
+                                    const uint16_t row_len,
+                                    const int8_t *filter_data,
+                                    const int32_t filter_offset,
+                                    const int32_t *bias,
+                                    int8_t *out_data,
+                                    const uint16_t out_channels,
+                                    const int32_t out_offset,
+                                    const int32_t out_shift,
+                                    const int32_t out_mult,
+                                    const int32_t activation_min,
+                                    const int32_t activation_max);
+
+/**
+ * @brief   Get scratch buffer size needed by softmax function
+ *
+ * @param   width
+ * @param   height
+ * @return  size in bytes
+ *
+ * @note    buffer must be 4 byte aligned
+ */
+int32_t esp_nn_get_softmax_scratch_size_ansi(const int32_t width, const int32_t height);
+
+/* ANSI C function to be hooked up when optimised version needed */
+int32_t esp_nn_get_softmax_scratch_size_opt(const int32_t width, const int32_t height);
+
+/**
+ * @brief   Set scratch buffer to be used by softmax function
+ *
+ * @param   buffer  this can be NULL if one needs to unset it
+ *                  must be aligned to 4 bytes
+ */
+void esp_nn_set_softmax_scratch_buf_ansi(void *buffer);
+
+/**
+ * @brief       reference softmax function
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ */
+void esp_nn_softmax_s8_ansi(const int8_t *input_data,
+                            const int32_t height,
+                            const int32_t width,
+                            const int32_t mult,
+                            const int32_t shift,
+                            const int32_t diff_min,
+                            int8_t *output_data);
+
+
+//////////////////////////// Generic optimisations /////////////////////////////
+
+/************************** Convolution functions *****************************/
+
+/**
+ * @brief       2d-convolution channelwise optimized version
+ *
+ * @note        operation: result += (input + offset) * filter
+ *
+ *              inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ */
+void esp_nn_conv_s8_opt(const data_dims_t *input_dims,
+                        const int8_t *input_data,
+                        const data_dims_t *filter_dims,
+                        const int8_t *filter_data,
+                        const int32_t *bias,
+                        const data_dims_t *output_dims,
+                        int8_t *out_data,
+                        const conv_params_t *conv_params,
+                        const quant_data_t *quant_data);
+
+/**
+ * @brief       depthwise convolution per channel optimized version
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              Version used in tflite is per channel.
+ *              This version follows the same footsprints.
+ *              Meaning, it has per out_channel shift and multiplier for
+ *              requantization
+ *
+ *              optimization notes: Though input_offset is int32 type,
+ *              offset values are contained in 8 bits [-128, 127]
+ */
+void esp_nn_depthwise_conv_s8_opt(const data_dims_t *input_dims,
+                                  const int8_t *input_data,
+                                  const data_dims_t *filter_dims,
+                                  const int8_t *filter_data,
+                                  const int32_t *bias,
+                                  const data_dims_t *output_dims,
+                                  int8_t *out_data,
+                                  const dw_conv_params_t *conv_params,
+                                  const quant_data_t *quant_data);
+
+int esp_nn_get_conv_scratch_size_opt(const data_dims_t *input_dims,
+                                     const data_dims_t *filter_dims,
+                                     const data_dims_t *output_dims,
+                                     const conv_params_t *conv_params);
+void esp_nn_set_conv_scratch_buf_opt(const void *buf);
+
+int esp_nn_get_depthwise_conv_scratch_size_opt(const data_dims_t *input_dims,
+                                               const data_dims_t *filter_dims,
+                                               const data_dims_t *output_dims,
+                                               const dw_conv_params_t *conv_params);
+void esp_nn_set_depthwise_conv_scratch_buf_opt(const void *buf);
+
+/* ANSI C function to be hooked up when optimised version needed */
+void esp_nn_set_softmax_scratch_buf_opt(void *buffer);
+
+/**
+ * @brief       optimised version of softmax function
+ *
+ * @note        the function uses extra buffer (4 * width bytes)
+ *              hence, scratch buffers must be set before calling this.
+ */
+void esp_nn_softmax_s8_opt(const int8_t *input_data,
+                           const int32_t height,
+                           const int32_t width,
+                           const int32_t mult,
+                           const int32_t shift,
+                           const int32_t diff_min,
+                           int8_t *output_data);

code/components/esp-nn/include/esp_nn_defs.h

@@ -0,0 +1,83 @@
+// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stdint.h>
+
+/**
+ * @brief structure to club data dims
+ * this structure can be used for input, output and filter
+ */
+typedef struct data_dims {
+    int32_t width;
+    int32_t height;
+    int32_t channels;
+
+    int32_t extra; // can be used as batch or any other param
+} data_dims_t;
+
+/**
+ * @brief 2d data structure (width, height)
+ *
+ */
+typedef struct data_2d {
+    int32_t width;
+    int32_t height;
+} data_2d_t;
+
+/**
+ * @brief min/max activation
+ */
+typedef struct act_params {
+    int32_t min;
+    int32_t max;
+} act_params_t;
+
+/**
+ * @brief per channel quant data
+ *
+ * @note number of shift and mult elements are equal to output channels
+ */
+typedef struct quant_data {
+    int32_t *shift;
+    int32_t *mult;
+} quant_data_t;
+
+/**
+ * @brief params specific to convolution 2d
+ *
+ */
+typedef struct conv_params {
+    int32_t in_offset;
+    int32_t out_offset;
+    data_2d_t stride;
+    data_2d_t padding;
+    data_2d_t dilation;
+    act_params_t activation;
+} conv_params_t;
+
+/**
+ * @brief params specific to depthwise convolution 2d
+ *
+ */
+typedef struct dw_conv_params {
+    int32_t in_offset;
+    int32_t out_offset;
+    int32_t ch_mult; // channel multiplier. (in_ch * ch_mult = out_ch)
+    data_2d_t stride;
+    data_2d_t padding;
+    data_2d_t dilation;
+    act_params_t activation;
+} dw_conv_params_t;

code/components/esp-nn/include/esp_nn_esp32s3.h

@@ -0,0 +1,231 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file        Header definitions to include for esp_nn optimized functions for
+ *              the ESP32-S3 platform
+ */
+
+#pragma once
+
+#include "esp_nn_defs.h"
+#include "esp_nn_ansi_headers.h"
+
+/************************** Basic math functions *****************************/
+
+
+/**
+ * @brief       elementwise addition
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ *
+ *              shift values are expected to be <= 0
+ */
+void esp_nn_add_elementwise_s8_esp32s3(const int8_t *input1_data,
+                                       const int8_t *input2_data,
+                                       const int32_t input1_offset,
+                                       const int32_t input2_offset,
+                                       const int32_t input1_mult,
+                                       const int32_t input2_mult,
+                                       const int32_t input1_shift,
+                                       const int32_t input2_shift,
+                                       const int32_t left_shift,
+                                       int8_t *output,
+                                       const int32_t out_offset,
+                                       const int32_t out_mult,
+                                       const int32_t out_shift,
+                                       const int32_t activation_min,
+                                       const int32_t activation_max,
+                                       const int32_t size);
+
+/**
+ * @brief       elementwise multiplication
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ *
+ *              output shift is expected to be <= 0
+ */
+void esp_nn_mul_elementwise_s8_esp32s3(const int8_t *input1_data,
+                                       const int8_t *input2_data,
+                                       const int32_t input1_offset,
+                                       const int32_t input2_offset,
+                                       int8_t *output,
+                                       const int32_t out_offset,
+                                       const int32_t out_mult,
+                                       const int32_t out_shift,
+                                       const int32_t activation_min,
+                                       const int32_t activation_max,
+                                       const int32_t size);
+
+
+/************************** Convolution functions *****************************/
+
+/**
+ * @brief       depthwise convolution per channel
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              Version used in tflite is per channel.
+ *              This version follows the same footsprints.
+ *              Meaning, it has per out_channel shift and multiplier for
+ *              requantization
+ *
+ *              optimization notes: Though input_offset is int32 type,
+ *              offset values are contained in 8 bits [-128, 127]
+ */
+void esp_nn_depthwise_conv_s8_esp32s3(const data_dims_t *input_dims,
+                                      const int8_t *input_data,
+                                      const data_dims_t *filter_dims,
+                                      const int8_t *filter_data,
+                                      const int32_t *bias,
+                                      const data_dims_t *output_dims,
+                                      int8_t *output_data,
+                                      const dw_conv_params_t *conv_params,
+                                      const quant_data_t *quant_data);
+
+/**
+ * @brief       2d - convolution channelwise
+ *
+ * @note        operation: result += (input + offset) * filter
+ *
+ *              inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ */
+void esp_nn_conv_s8_esp32s3(const data_dims_t *input_dims,
+                            const int8_t *input_data,
+                            const data_dims_t *filter_dims,
+                            const int8_t *filter_data,
+                            const int32_t *bias,
+                            const data_dims_t *output_dims,
+                            int8_t *output_data,
+                            const conv_params_t *conv_params,
+                            const quant_data_t *quant_data);
+
+int esp_nn_get_conv_scratch_size_esp32s3(const data_dims_t *input_dims,
+                                         const data_dims_t *filter_dims,
+                                         const data_dims_t *output_dims,
+                                         const conv_params_t *conv_params);
+void esp_nn_set_conv_scratch_buf_esp32s3(const void *buf);
+
+int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const data_dims_t *input_dims,
+                                                   const data_dims_t *filter_dims,
+                                                   const data_dims_t *output_dims,
+                                                   const dw_conv_params_t *conv_params);
+void esp_nn_set_depthwise_conv_scratch_buf_esp32s3(const void *buf);
+
+/************************** Pooling functions *****************************/
+
+/**
+ * @brief       max_pool
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ */
+void esp_nn_max_pool_s8_esp32s3(const int8_t *input,
+                                const uint16_t input_wd,
+                                const uint16_t input_ht,
+                                int8_t *output,
+                                const uint16_t output_wd,
+                                const uint16_t output_ht,
+                                const uint16_t stride_wd,
+                                const uint16_t stride_ht,
+                                const uint16_t filter_wd,
+                                const uint16_t filter_ht,
+                                const uint16_t pad_wd,
+                                const uint16_t pad_ht,
+                                const int32_t activation_min,
+                                const int32_t activation_max,
+                                const uint16_t channels);
+
+/**
+ * @brief       avg_pool
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ */
+void esp_nn_avg_pool_s8_esp32s3(const int8_t *input,
+                                const uint16_t input_wd,
+                                const uint16_t input_ht,
+                                int8_t *output,
+                                const uint16_t output_wd,
+                                const uint16_t output_ht,
+                                const uint16_t stride_wd,
+                                const uint16_t stride_ht,
+                                const uint16_t filter_wd,
+                                const uint16_t filter_ht,
+                                const uint16_t pad_wd,
+                                const uint16_t pad_ht,
+                                const int32_t activation_min,
+                                const int32_t activation_max,
+                                const uint16_t channels);
+
+
+/************************** Fully connected functions *****************************/
+
+/**
+ * @brief       fully connected
+ *
+ * @note        inputs type: int8_t, output: int8_t
+ *              input offsets: although int32_t, they are contained in 8 bits [-128, 127]
+ *
+ *              Current version works only on aligned input.
+ *              row_len and channels should both be multiple of 8.
+ */
+void esp_nn_fully_connected_s8_esp32s3(const int8_t *input_data,
+                                       const int32_t input_offset,
+                                       const uint16_t row_len,
+                                       const int8_t *filter_data,
+                                       const int32_t filter_offset,
+                                       const int32_t *bias,
+                                       int8_t *out_data,
+                                       const uint16_t out_channels,
+                                       const int32_t out_offset,
+                                       const int32_t out_shift,
+                                       const int32_t out_mult,
+                                       const int32_t activation_min,
+                                       const int32_t activation_max);
+
+/**
+ * @brief       relu6
+ *
+ * @note        inout: int8_t
+ */
+void esp_nn_relu6_s8_esp32s3(int8_t *data, uint16_t size);
+
+/********************** function defines ***************************/
+
+#define esp_nn_add_elementwise_s8 esp_nn_add_elementwise_s8_esp32s3
+#define esp_nn_mul_elementwise_s8 esp_nn_mul_elementwise_s8_esp32s3
+
+#define esp_nn_depthwise_conv_s8 esp_nn_depthwise_conv_s8_esp32s3
+
+#define esp_nn_get_conv_scratch_size esp_nn_get_conv_scratch_size_esp32s3
+#define esp_nn_set_conv_scratch_buf esp_nn_set_conv_scratch_buf_esp32s3
+
+#define esp_nn_get_depthwise_conv_scratch_size esp_nn_get_depthwise_conv_scratch_size_esp32s3
+#define esp_nn_set_depthwise_conv_scratch_buf esp_nn_set_depthwise_conv_scratch_buf_esp32s3
+
+#define esp_nn_conv_s8 esp_nn_conv_s8_esp32s3
+
+#define esp_nn_relu6_s8 esp_nn_relu6_s8_esp32s3
+
+#define esp_nn_avg_pool_s8 esp_nn_avg_pool_s8_esp32s3
+#define esp_nn_max_pool_s8 esp_nn_max_pool_s8_esp32s3
+
+#define esp_nn_fully_connected_s8 esp_nn_fully_connected_s8_esp32s3
+
+#define esp_nn_get_softmax_scratch_size esp_nn_get_softmax_scratch_size_opt
+#define esp_nn_set_softmax_scratch_buf esp_nn_set_softmax_scratch_buf_opt
+#define esp_nn_softmax_s8 esp_nn_softmax_s8_opt

code/components/esp-nn/include/esp_nn_generic_opt.h

@@ -0,0 +1,47 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file        Header definitions to include for esp_nn generic optimisations
+ *              For functions which not having optimisations, _ansi versions are picked.
+ */
+
+#pragma once
+
+#include "esp_nn_defs.h"
+#include "esp_nn_ansi_headers.h"
+
+#define esp_nn_add_elementwise_s8 esp_nn_add_elementwise_s8_ansi
+#define esp_nn_mul_elementwise_s8 esp_nn_mul_elementwise_s8_ansi
+
+#define esp_nn_depthwise_conv_s8 esp_nn_depthwise_conv_s8_opt
+
+#define esp_nn_conv_s8 esp_nn_conv_s8_opt
+
+#define esp_nn_get_conv_scratch_size esp_nn_get_conv_scratch_size_opt
+#define esp_nn_set_conv_scratch_buf esp_nn_set_conv_scratch_buf_opt
+
+#define esp_nn_get_depthwise_conv_scratch_size esp_nn_get_depthwise_conv_scratch_size_opt
+#define esp_nn_set_depthwise_conv_scratch_buf esp_nn_set_depthwise_conv_scratch_buf_opt
+
+#define esp_nn_relu6_s8 esp_nn_relu6_s8_ansi
+
+#define esp_nn_avg_pool_s8 esp_nn_avg_pool_s8_ansi
+#define esp_nn_max_pool_s8 esp_nn_max_pool_s8_ansi
+
+#define esp_nn_fully_connected_s8 esp_nn_fully_connected_s8_ansi
+
+#define esp_nn_get_softmax_scratch_size esp_nn_get_softmax_scratch_size_opt
+#define esp_nn_set_softmax_scratch_buf esp_nn_set_softmax_scratch_buf_opt
+#define esp_nn_softmax_s8 esp_nn_softmax_s8_opt

code/components/esp-nn/src/activation_functions/esp_nn_relu_ansi.c

@@ -0,0 +1,30 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <common_functions.h>
+
+void esp_nn_relu6_s8_ansi(int8_t *data, uint16_t size)
+{
+    int32_t i;
+
+    for (i = 0; i < size; i++) {
+        int32_t ip = data[i];
+
+        ip = max(ip, 0);
+        data[i] = min(ip, 6);
+    }
+}

code/components/esp-nn/src/basic_math/esp_nn_add_ansi.c

@@ -0,0 +1,97 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+
+#include <common_functions.h>
+
+void esp_nn_add_elementwise_u8_ansi(const uint8_t *input1_data,
+                                    const uint8_t *input2_data,
+                                    const int32_t input1_offset,
+                                    const int32_t input2_offset,
+                                    const int32_t input1_mult,
+                                    const int32_t input2_mult,
+                                    const int32_t input1_shift,
+                                    const int32_t input2_shift,
+                                    const int32_t left_shift,
+                                    uint8_t *output,
+                                    const int32_t out_offset,
+                                    const int32_t out_mult,
+                                    const int32_t out_shift,
+                                    const int32_t activation_min,
+                                    const int32_t activation_max,
+                                    const int32_t size)
+{
+    for (int i = 0; i < size; i++) {
+        int32_t tmp1 = input1_data[i] + input1_offset;
+        int32_t tmp2 = input2_data[i] + input2_offset;
+
+        tmp1 <<= left_shift;
+        tmp2 <<= left_shift;
+
+        tmp1 = esp_nn_sat_round_doubling_high_mul(tmp1, input1_mult);
+        tmp2 = esp_nn_sat_round_doubling_high_mul(tmp2, input2_mult);
+
+        tmp1 = esp_nn_div_by_power_of_two(tmp1, -input1_shift);
+        tmp2 = esp_nn_div_by_power_of_two(tmp2, -input2_shift);
+
+        int32_t out = tmp1 + tmp2;
+        out = esp_nn_sat_round_doubling_high_mul(out, out_mult);
+        out = esp_nn_div_by_power_of_two(out, -out_shift);
+        out = out + out_offset;
+
+        out = max(activation_min, min(out, activation_max));
+        output[i] = (uint8_t) out;
+    }
+}
+
+void esp_nn_add_elementwise_s8_ansi(const int8_t *input1_data,
+                                    const int8_t *input2_data,
+                                    const int32_t input1_offset,
+                                    const int32_t input2_offset,
+                                    const int32_t input1_mult,
+                                    const int32_t input2_mult,
+                                    const int32_t input1_shift,
+                                    const int32_t input2_shift,
+                                    const int32_t left_shift,
+                                    int8_t *output,
+                                    const int32_t out_offset,
+                                    const int32_t out_mult,
+                                    const int32_t out_shift,
+                                    const int32_t activation_min,
+                                    const int32_t activation_max,
+                                    const int32_t size)
+{
+    for (int i = 0; i < size; i++) {
+        int32_t tmp1 = input1_data[i] + input1_offset;
+        int32_t tmp2 = input2_data[i] + input2_offset;
+
+        tmp1 <<= left_shift;
+        tmp2 <<= left_shift;
+
+        tmp1 = esp_nn_sat_round_doubling_high_mul(tmp1, input1_mult);
+        tmp2 = esp_nn_sat_round_doubling_high_mul(tmp2, input2_mult);
+
+        tmp1 = esp_nn_div_by_power_of_two(tmp1, -input1_shift);
+        tmp2 = esp_nn_div_by_power_of_two(tmp2, -input2_shift);
+
+        int32_t out = tmp1 + tmp2;
+        out = esp_nn_sat_round_doubling_high_mul(out, out_mult);
+        out = esp_nn_div_by_power_of_two(out, -out_shift);
+        out = out + out_offset;
+
+        out = max(activation_min, min(out, activation_max));
+        output[i] = (int8_t) out;
+    }
+}

code/components/esp-nn/src/basic_math/esp_nn_mul_ansi.c

@@ -0,0 +1,42 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+
+#include <common_functions.h>
+
+void esp_nn_mul_elementwise_s8_ansi(const int8_t *input1_data,
+                                    const int8_t *input2_data,
+                                    const int32_t input1_offset,
+                                    const int32_t input2_offset,
+                                    int8_t *output,
+                                    const int32_t out_offset,
+                                    const int32_t out_mult,
+                                    const int32_t out_shift,
+                                    const int32_t activation_min,
+                                    const int32_t activation_max,
+                                    const int32_t size)
+{
+    for (int i = 0; i < size; i++) {
+        int32_t tmp1 = input1_data[i] + input1_offset;
+        int32_t tmp2 = input2_data[i] + input2_offset;
+
+        int32_t out = tmp1 * tmp2;
+        out = esp_nn_multiply_by_quantized_mult(out, out_mult, out_shift);
+        out = out + out_offset;
+
+        out = max(activation_min, min(out, activation_max));
+        output[i] = (int8_t) out;
+    }
+}

code/components/esp-nn/src/common/common_functions.h

@@ -0,0 +1,255 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+
+/**
+ * c99 standard still doesn't strictly inline functions
+ * We need to use attribute as well to do this.
+ */
+#define __NN_FORCE_INLINE__ __attribute((always_inline)) static inline
+
+/* min/max macros */
+#ifndef max
+#define max(a, b) ({            \
+    __typeof__ (a) _a = (a);    \
+    __typeof__ (b) _b = (b);    \
+    _a > _b ? _a : _b;          \
+})
+
+#define min(a, b) ({            \
+    __typeof__ (a) _a = (a);    \
+    __typeof__ (b) _b = (b);    \
+    _a < _b ? _a : _b;          \
+})
+#endif
+
+__NN_FORCE_INLINE__ int32_t esp_nn_clz32(uint32_t in)
+{
+#if CONFIG_IDF_TARGET_ARCH_XTENSA
+    __asm__ volatile("nsau %0, %0" : "+r" (in));
+    return in;
+#elif defined(__GNUC__)
+    return __builtin_clz(in);
+#else
+    int32_t count = 32;
+    uint32_t x = in, y = in >> 16;
+    if (y != 0) {
+        count -= 16;
+        x = y;
+    }
+    y = x >> 8;
+    if (y != 0) {
+        count -= 8;
+        x = y;
+    }
+    y = x >> 4;
+    if (y != 0) {
+        count -= 4;
+        x = y;
+    }
+    y = x >> 2;
+    if (y != 0) {
+        count -= 2;
+        x = y;
+    }
+    y = x >> 1;
+    if (y != 0) {
+        return count - 2;
+    }
+    return count - x;
+#endif
+}
+
+/**
+ * Signed saturate a 32 bit value to 8 bits keeping output in 32 bit variable.
+ */
+__NN_FORCE_INLINE__ int32_t esp_nn_saturate8(int32_t in)
+{
+#if CONFIG_IDF_TARGET_ARCH_XTENSA
+    __asm__ volatile("clamps %0, %0, 7" : "+a"(in));
+    return in;
+#else
+    return max(INT8_MIN, min(in, INT8_MAX));
+#endif
+}
+
+__NN_FORCE_INLINE__ int32_t esp_nn_pick_sat_high32_of64(int64_t val64)
+{
+    int32_t sign = (int32_t) (val64 >> 63);
+    int32_t to_add = sign & ((1ul << 31) - 1);
+    return (int32_t) ((int64_t) (val64 + to_add) >> 31);
+}
+
+__NN_FORCE_INLINE__ int32_t esp_nn_sat_round_doubling_high_mul(int32_t in0, int32_t in1)
+{
+    int32_t result;
+    int64_t in0_64 = (int64_t) in0;
+    bool overflow = (in0 == in1) && (in0 == (int32_t) INT32_MIN);
+
+    /* Nudge value */
+    int64_t nudge_val = 1 << 30;
+    if ((in0 < 0) ^ (in1 < 0)) {
+        nudge_val = 1 - nudge_val;
+    }
+
+    /* Multiply and add nudge */
+    int64_t mult = in0_64 * in1 + nudge_val;
+
+    /* Round and pickup 32 bits */
+    result = esp_nn_pick_sat_high32_of64(mult);
+
+    return overflow ? INT32_MAX : result;
+}
+
+/**
+ * fast version
+ * this will fail for values closer to INT32_MAX and INT32_MIN by `1 << (exponent - 1)`.
+ * We can afford to do this because we are at the very last stage of filter.
+ * Also it is pretty rare condition as our output is going to be 8 bit.
+ */
+__NN_FORCE_INLINE__ int32_t esp_nn_div_by_power_of_two_fast(int32_t val, int32_t exponent)
+{
+    int32_t to_add = (1 << (exponent - 1)) - (val < 0);
+    return (int32_t) ((val + to_add) >> exponent);
+}
+
+__NN_FORCE_INLINE__ int32_t esp_nn_div_by_power_of_two(int32_t val, int32_t exponent)
+{
+    int32_t result;
+
+    const int32_t mask = (1 << exponent) - 1;
+    const int32_t remainder = val & mask;
+
+    result = val >> exponent;
+    int32_t threshold = (mask >> 1) + (result < 0);
+
+    if (remainder > threshold) {
+        result += 1;
+    }
+    return result;
+}
+
+__NN_FORCE_INLINE__ int32_t esp_nn_multiply_by_quantized_mult(int32_t x, int32_t mult, int32_t shift)
+{
+    int32_t left_shift = shift > 0 ? shift : 0;
+    int32_t right_shift = shift > 0 ? 0 : -shift;
+    int32_t result = esp_nn_sat_round_doubling_high_mul(x * (1 << left_shift), mult);
+    return esp_nn_div_by_power_of_two(result, right_shift);
+}
+
+__NN_FORCE_INLINE__ int32_t esp_nn_multiply_by_quantized_mult_fast(int32_t x, int32_t mult, int32_t shift)
+{
+    int32_t left_shift = max(shift, 0);
+    int32_t right_shift = left_shift - shift;
+
+    int64_t nudge_val = 1 << 30;
+    int64_t in0_64 = (int64_t) (x << left_shift);
+
+    /* Multiply and add nudge */
+    int64_t mult_64 = in0_64 * mult + nudge_val;
+    int32_t result = (int32_t) (mult_64 >> 31);
+    if (right_shift) {
+        result = esp_nn_div_by_power_of_two_fast(result, right_shift);
+    }
+    return result;
+}
+
+static void esp_nn_aligned_s8_pad_with_value(const int8_t *src, int8_t *dst,
+                                             const uint16_t input_wd,
+                                             const uint16_t input_ht,
+                                             const uint16_t channels,
+                                             const int32_t pad_val,
+                                             const uint16_t pad_wd,
+                                             const uint16_t pad_ht)
+{
+    /* memset with pad_val */
+    memset(dst, pad_val, ((input_wd + 2 * pad_wd) * (input_ht + 2 * pad_ht)) * channels);
+    dst += (pad_wd + input_wd + pad_wd) * channels;
+
+    for (int i = 0; i < input_ht; i++) {
+        dst += pad_wd * channels;
+        for (int j = 0; j < input_wd * channels; j++) {
+            *dst++ = *src++;
+        }
+        dst += pad_wd * channels;
+    }
+}
+
+static void esp_nn_aligned_s8_pad_end_with_value(const int8_t *src, int8_t *dst,
+                                                 const uint16_t input_wd,
+                                                 const uint16_t input_ht,
+                                                 const uint16_t channels,
+                                                 const int32_t pad_val,
+                                                 const uint16_t pad_wd,
+                                                 const uint16_t pad_ht)
+{
+    for (int i = 0; i < input_ht; i++) {
+        for (int j = 0; j < input_wd * channels; j++) {
+            *dst++ = *src++;
+        }
+        if (pad_wd) {
+            memset(dst, pad_val, pad_wd * channels);
+            dst += pad_wd * channels;
+        }
+    }
+    /* pad end `pad_ht` lines at end */
+    if (pad_ht) {
+        memset(dst, pad_val, (input_wd + pad_wd) * pad_ht * channels);
+    }
+}
+
+/**
+ * @brief       convert 8 bit input data to 16 bit
+ *
+ * @param       src int8_t source data
+ * @param       dst int16_t dst data
+ * @param       size length of data
+ * @param       offset  offset to be added to src data. Range: [-128, 127]
+ */
+__NN_FORCE_INLINE__ void esp_nn_s8_to_s16_with_offset(const int8_t *src, int16_t *dst,
+                                                      const int size, const int32_t offset)
+{
+    int i = 0;
+    for (; i < size; i += 2) {
+        dst[i + 0] = src[i + 0] + offset;
+        dst[i + 1] = src[i + 1] + offset;
+    }
+    if(i < size) {
+        dst[i] = src[i] + offset;
+    }
+}
+
+/**
+ * @brief       convert 8 bit input data to 16 bit
+ *
+ * @param       src int8_t source data
+ * @param       dst int16_t dst data
+ * @param       size length of data
+ */
+__NN_FORCE_INLINE__ void esp_nn_s8_to_s16(const int8_t *src, int16_t *dst, const int size)
+{
+    int i = 0;
+    for (; i < size; i += 2) {
+        dst[i + 0] = src[i + 0];
+        dst[i + 1] = src[i + 1];
+    }
+    if(i < size) {
+        dst[i] = src[i];
+    }
+}

code/components/esp-nn/src/convolution/esp_nn_conv_ansi.c

@@ -0,0 +1,179 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <esp_nn_defs.h>
+
+#include <common_functions.h>
+
+int esp_nn_get_conv_scratch_size_ansi(const data_dims_t *input_dims,
+                                      const data_dims_t *filter_dims,
+                                      const data_dims_t *output_dims,
+                                      const conv_params_t *conv_params)
+{
+    return 0;
+}
+
+void esp_nn_set_conv_scratch_buf_ansi(const void *buf)
+{
+
+}
+
+/**
+ * Assumption 1: i/p channels == o/p channels
+ * Assumption 2: Pointers are valid
+ * Assumption 3: dialation width = 1
+ */
+void esp_nn_conv_u8_ansi(const uint8_t *input_data,
+                         const uint16_t input_wd,
+                         const uint16_t input_ht,
+                         const uint16_t in_channels,
+                         const int32_t input_offset,
+                         const uint16_t pad_wd,
+                         const uint16_t pad_ht,
+                         const uint16_t stride_wd,
+                         const uint16_t stride_ht,
+                         const uint8_t *filter_data,
+                         const uint16_t filter_wd,
+                         const uint16_t filter_ht,
+                         const int32_t filter_offset,
+                         const int32_t *bias,
+                         uint8_t *out_data,
+                         const uint16_t out_wd,
+                         const uint16_t out_ht,
+                         const uint16_t out_channels,
+                         const int32_t out_offset,
+                         const int32_t out_shift,
+                         const int32_t out_mult,
+                         const int32_t activation_min,
+                         const int32_t activation_max)
+{
+    for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
+        const int16_t base_y = (out_y * stride_ht) - pad_ht;
+        for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
+            const int16_t base_x = (out_x * stride_wd) - pad_wd;
+            for (int out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {//channel_loop
+                int32_t result = 0;
+
+                /* Select filter so as the point doesn't lie outside block */
+                int filter_y_start = max(0, -base_y);
+                int filter_x_start = max(0, -base_x);
+                int filter_y_end = min(filter_ht, input_ht - base_y);
+                int filter_x_end = min(filter_wd, input_wd - base_x);
+
+                for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                    const int32_t idx_y = base_y + filter_y_idx;
+                    for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                        const int32_t idx_x = base_x + filter_x_idx;
+                        for (int in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
+                            int32_t input_index = (idx_y * input_wd + idx_x) * in_channels + in_ch_idx;
+                            int32_t filter_index = ((out_ch_idx * filter_ht + filter_y_idx)
+                                                    * filter_wd + filter_x_idx) * in_channels
+                                                   + in_ch_idx;
+                            int32_t input_val = input_data[input_index] + input_offset;
+                            int32_t filter_val = filter_data[filter_index] + filter_offset;
+                            result += input_val * filter_val;
+                        }
+                    }
+                }
+                if (bias) {
+                    result += bias[out_ch_idx];
+                }
+                result = esp_nn_multiply_by_quantized_mult(result, out_mult, out_shift);
+                result += out_offset;
+                result = max(result, activation_min);
+                result = min(result, activation_max);
+
+                int out_index = (out_y * out_wd + out_x) * out_channels + out_ch_idx;
+                out_data[out_index] = (uint8_t) result;
+            }
+        }
+    }
+}
+
+/**
+ * Assumption 1: i/p channels == o/p channels
+ * Assumption 2: Pointers are valid
+ * Assumption 3: dialation width = 1
+ */
+void esp_nn_conv_s8_ansi(const data_dims_t *input_dims,
+                         const int8_t *input_data,
+                         const data_dims_t *filter_dims,
+                         const int8_t *filter_data,
+                         const int32_t *bias,
+                         const data_dims_t *output_dims,
+                         int8_t *out_data,
+                         const conv_params_t *conv_params,
+                         const quant_data_t *quant_data)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t in_channels = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const uint16_t out_channels = output_dims->channels;
+    const int32_t *out_shift = quant_data->shift;
+    const int32_t *out_mult = quant_data->mult;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+
+    int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
+
+    for (out_y = 0; out_y < out_ht; out_y++) {
+        for (out_x = 0; out_x < out_wd; out_x++) {
+            for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
+                int32_t conv_out = 0;
+
+                const int32_t base_y = stride_ht * out_y - pad_ht;
+                const int32_t base_x = stride_wd * out_x - pad_wd;
+
+                const int32_t filter_y_start = max(0, -base_y);
+                const int32_t filter_x_start = max(0, -base_x);
+
+                const int32_t filter_y_end = min(filter_ht, input_ht - base_y);
+                const int32_t filter_x_end = min(filter_wd, input_wd - base_x);
+
+                for (filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                    for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                        const int32_t in_row = base_y + filter_y_idx;
+                        const int32_t in_col = base_x + filter_x_idx;
+                        int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels;
+                        int32_t filter_base_offset = out_ch_idx * in_channels * filter_ht * filter_wd +
+                                                       (filter_y_idx * filter_wd + filter_x_idx) * in_channels;
+                        for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
+                            conv_out +=
+                                (input_data[input_base_offset + in_ch_idx] + input_offset) *
+                                filter_data[filter_base_offset + in_ch_idx];
+                        }
+                    }
+                }
+                if (bias) {
+                    conv_out += bias[out_ch_idx];
+                }
+                conv_out = esp_nn_multiply_by_quantized_mult(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
+                conv_out += out_offset;
+                conv_out = max(conv_out, activation_min);
+                conv_out = min(conv_out, activation_max);
+                *out_data++ = (int8_t) conv_out;
+            }
+        }
+    }
+}

code/components/esp-nn/src/convolution/esp_nn_conv_esp32s3.c

@@ -0,0 +1,463 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+#include <esp_nn_defs.h>
+
+#include <common_functions.h>
+
+static int16_t *scratch_buffer = NULL;
+
+extern void esp_nn_conv_s8_mult8_1x1_esp32s3(const int8_t *input_data,
+                                             const uint16_t input_wd,
+                                             const uint16_t input_ht,
+                                             const uint16_t in_channels,
+                                             const int32_t input_offset,
+                                             const int8_t *filter_aligned,
+                                             const int32_t *bias,
+                                             int8_t *out_data,
+                                             const uint16_t out_wd,
+                                             const uint16_t out_ht,
+                                             const uint16_t out_channels,
+                                             const int32_t out_offset,
+                                             const int32_t *out_shift,
+                                             const int32_t *out_mult,
+                                             const int32_t activation_min,
+                                             const int32_t activation_max,
+                                             void *buffer /* scratch buffer */);
+
+extern void esp_nn_conv_s16_mult4_1x1_esp32s3(const int16_t *input_data,
+                                              const uint16_t input_wd,
+                                              const uint16_t input_ht,
+                                              const uint16_t in_channels,
+                                              const int16_t *filter_data,
+                                              const int32_t *bias,
+                                              int8_t *out_data,
+                                              const uint16_t out_wd,
+                                              const uint16_t out_ht,
+                                              const uint16_t out_channels,
+                                              const int32_t out_offset,
+                                              const int32_t *out_shift,
+                                              const int32_t *out_mult,
+                                              const int32_t activation_min,
+                                              const int32_t activation_max,
+                                              void *buffer /* scratch buffer */);
+
+extern void esp_nn_conv_s16_mult8_esp32s3(const int16_t *input_data,
+                                          const uint16_t input_wd,
+                                          const uint16_t input_ht,
+                                          const uint16_t in_channels,
+                                          const uint16_t pad_wd,
+                                          const uint16_t pad_ht,
+                                          const uint16_t stride_wd,
+                                          const uint16_t stride_ht,
+                                          const int16_t *filter_data,
+                                          const uint16_t filter_wd,
+                                          const uint16_t filter_ht,
+                                          const int32_t *bias,
+                                          int8_t *out_data,
+                                          const uint16_t out_wd,
+                                          const uint16_t out_ht,
+                                          const uint16_t out_channels,
+                                          const int32_t out_offset,
+                                          const int32_t *out_shift,
+                                          const int32_t *out_mult,
+                                          const int32_t activation_min,
+                                          const int32_t activation_max);
+
+extern void esp_nn_aligned_s8_to_s16_with_offset_esp32s3(const int8_t *src, int16_t *dst,
+                                                         const int size, const int32_t offset);
+
+extern void esp_nn_s8_to_s16_esp32s3(const int8_t *src, int16_t *dst, const int size);
+
+static void esp_nn_conv_s8_unrolled(const data_dims_t *input_dims,
+                                    const int8_t *input_data,
+                                    const data_dims_t *filter_dims,
+                                    const int8_t *filter_data,
+                                    const int32_t *bias,
+                                    const data_dims_t *output_dims,
+                                    int8_t *out_data,
+                                    const conv_params_t *conv_params,
+                                    const quant_data_t *quant_data)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t in_ch = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const uint16_t out_ch = output_dims->channels;
+    const int32_t *out_shift = quant_data->shift;
+    const int32_t *out_mult = quant_data->mult;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+
+    int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
+
+    for (out_y = 0; out_y < out_ht; out_y++) {
+        for (out_x = 0; out_x < out_wd; out_x++) {
+            for (out_ch_idx = 0; out_ch_idx < out_ch; out_ch_idx++) {
+                int32_t conv_out = 0;
+
+                const int32_t base_y = stride_ht * out_y - pad_ht;
+                const int32_t base_x = stride_wd * out_x - pad_wd;
+
+                const int32_t filter_y_start = max(0, -base_y);
+                const int32_t filter_x_start = max(0, -base_x);
+
+                const int32_t filter_y_end = min(filter_ht, input_ht - base_y);
+                const int32_t filter_x_end = min(filter_wd, input_wd - base_x);
+
+                for (filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                    for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                        const int32_t in_row = base_y + filter_y_idx;
+                        const int32_t in_col = base_x + filter_x_idx;
+                        int32_t input_base_offset = (in_row * input_wd + in_col) * in_ch;
+                        int32_t filter_base_offset = out_ch_idx * in_ch * filter_ht * filter_wd +
+                                                       (filter_y_idx * filter_wd + filter_x_idx) * in_ch;
+                        for (in_ch_idx = 0; in_ch_idx < in_ch; in_ch_idx++) {
+                            conv_out +=
+                                (input_data[input_base_offset + in_ch_idx] + input_offset) *
+                                filter_data[filter_base_offset + in_ch_idx];
+                        }
+                    }
+                }
+                if (bias) {
+                    conv_out += bias[out_ch_idx];
+                }
+                conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
+                conv_out += out_offset;
+                conv_out = max(conv_out, activation_min);
+                conv_out = min(conv_out, activation_max);
+                *out_data++ = (int8_t) conv_out;
+            }
+        }
+    }
+}
+
+static void esp_nn_conv_s8_pad_valid(const int8_t *input_data,
+                                     const uint16_t input_wd,
+                                     const uint16_t input_ht,
+                                     const uint16_t in_channels,
+                                     const int32_t input_offset,
+                                     const uint16_t stride_wd,
+                                     const uint16_t stride_ht,
+                                     const int8_t *filter_data,
+                                     const uint16_t filter_wd,
+                                     const uint16_t filter_ht,
+                                     const int32_t *bias,
+                                     int8_t *out_data,
+                                     const uint16_t out_wd,
+                                     const uint16_t out_ht,
+                                     const uint16_t out_channels,
+                                     const int32_t out_offset,
+                                     const int32_t *out_shift,
+                                     const int32_t *out_mult,
+                                     const int32_t activation_min,
+                                     const int32_t activation_max)
+{
+    int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
+
+    for (out_y = 0; out_y < out_ht; out_y++) {
+        for (out_x = 0; out_x < out_wd; out_x++) {
+            for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
+                int32_t conv_out = 0;
+
+                const int32_t base_y = stride_ht * out_y;
+                const int32_t base_x = stride_wd * out_x;
+
+                for (filter_y_idx = 0; filter_y_idx < filter_ht; filter_y_idx++) {
+                    for (filter_x_idx = 0; filter_x_idx < filter_wd; filter_x_idx++) {
+                        const int32_t in_row = base_y + filter_y_idx;
+                        const int32_t in_col = base_x + filter_x_idx;
+                        int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels;
+                        int32_t filter_base_offset = out_ch_idx * in_channels * filter_ht * filter_wd +
+                                                       (filter_y_idx * filter_wd + filter_x_idx) * in_channels;
+                        const int8_t *input_data_ptr = input_data + input_base_offset;
+                        const int8_t *filter_data_ptr = filter_data + filter_base_offset;
+                        for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
+                            conv_out += (*input_data_ptr++ + input_offset) * *filter_data_ptr++;
+                        }
+                    }
+                }
+                if (bias) {
+                    conv_out += bias[out_ch_idx];
+                }
+                conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
+                conv_out += out_offset;
+                conv_out = max(conv_out, activation_min);
+                conv_out = min(conv_out, activation_max);
+                *out_data++ = (int8_t) conv_out;
+            }
+        }
+    }
+}
+
+static void esp_nn_conv_s8_pad_valid_3x3(const int8_t *input_data,
+                                         const uint16_t input_wd,
+                                         const uint16_t input_ht,
+                                         const uint16_t in_channels,
+                                         const int32_t input_offset,
+                                         const uint16_t stride_wd,
+                                         const uint16_t stride_ht,
+                                         const int8_t *filter_data,
+                                         const int32_t *bias,
+                                         int8_t *out_data,
+                                         const uint16_t out_wd,
+                                         const uint16_t out_ht,
+                                         const uint16_t out_channels,
+                                         const int32_t out_offset,
+                                         const int32_t *out_shift,
+                                         const int32_t *out_mult,
+                                         const int32_t activation_min,
+                                         const int32_t activation_max)
+{
+    int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
+
+    for (out_y = 0; out_y < out_ht; out_y++) {
+        for (out_x = 0; out_x < out_wd; out_x++) {
+            const int32_t base_y = stride_ht * out_y;
+            const int32_t base_x = stride_wd * out_x;
+            for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
+                int32_t conv_out = 0;
+                for (filter_y_idx = 0; filter_y_idx < 3; filter_y_idx++) {
+                    for (filter_x_idx = 0; filter_x_idx < 3; filter_x_idx++) {
+                        const int32_t in_row = base_y + filter_y_idx;
+                        const int32_t in_col = base_x + filter_x_idx;
+                        int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels;
+                        int32_t filter_base_offset = out_ch_idx * in_channels * 3 * 3 +
+                                                       (filter_y_idx * 3 + filter_x_idx) * in_channels;
+                        const int8_t *input_data_ptr = input_data + input_base_offset;
+                        const int8_t *filter_data_ptr = filter_data + filter_base_offset;
+                        for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
+                            conv_out += (*input_data_ptr++ + input_offset) * *filter_data_ptr++;
+                        }
+                    }
+                }
+                if (bias) {
+                    conv_out += bias[out_ch_idx];
+                }
+                conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
+                conv_out += out_offset;
+                conv_out = max(conv_out, activation_min);
+                conv_out = min(conv_out, activation_max);
+                *out_data++ = (int8_t) conv_out;
+            }
+        }
+    }
+}
+
+static void esp_nn_conv_s8_pad_valid_ch3_3x3(const int8_t *input_data,
+                                             const uint16_t input_wd,
+                                             const uint16_t input_ht,
+                                             const int32_t input_offset,
+                                             const uint16_t stride_wd,
+                                             const uint16_t stride_ht,
+                                             const int8_t *filter_data,
+                                             const int32_t *bias,
+                                             int8_t *out_data,
+                                             const uint16_t out_wd,
+                                             const uint16_t out_ht,
+                                             const uint16_t out_channels,
+                                             const int32_t out_offset,
+                                             const int32_t *out_shift,
+                                             const int32_t *out_mult,
+                                             const int32_t activation_min,
+                                             const int32_t activation_max)
+{
+    int32_t out_ch_idx, out_y, out_x, filter_y_idx;
+
+    /* use scratch_buffer to pre-compute offset factor */
+    int16_t *filter_sum = (int16_t *) scratch_buffer;
+    const int8_t *filter_ptr = filter_data;
+    for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
+        int16_t sum_val = 0;
+        for (int i = 0; i < 9; i++) {
+            sum_val += *filter_ptr++;
+            sum_val += *filter_ptr++;
+            sum_val += *filter_ptr++;
+        }
+        *filter_sum++ = sum_val;
+    }
+
+    for (out_y = 0; out_y < out_ht; out_y++) {
+        for (out_x = 0; out_x < out_wd; out_x++) {
+            const int8_t *filter_data_ptr = filter_data;
+            const int32_t base_y = stride_ht * out_y;
+            const int32_t base_x = stride_wd * out_x;
+            const int8_t *input_base_ptr = input_data + (base_y * input_wd + base_x) * 3;
+            int16_t *filter_sum = (int16_t *) scratch_buffer;
+            for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
+                int32_t conv_out = 0;
+
+                for (filter_y_idx = 0; filter_y_idx < 3; filter_y_idx++) {
+                    const int8_t *input_data_ptr = input_base_ptr + (filter_y_idx * input_wd) * 3;
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+                    conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
+                }
+
+                conv_out += *filter_sum++ * input_offset;
+
+                if (bias) {
+                    conv_out += bias[out_ch_idx];
+                }
+                conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
+                conv_out += out_offset;
+                conv_out = max(conv_out, activation_min);
+                conv_out = min(conv_out, activation_max);
+                *out_data++ = (int8_t) conv_out;
+            }
+        }
+    }
+}
+
+int esp_nn_get_conv_scratch_size_esp32s3(const data_dims_t *input_dims,
+                                         const data_dims_t *filter_dims,
+                                         const data_dims_t *output_dims,
+                                         const conv_params_t *conv_params)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t in_ch = input_dims->channels;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t out_ch = output_dims->channels;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+
+    int filter_size = filter_wd * filter_ht * in_ch * out_ch;
+    int input_size = input_wd * input_ht * in_ch;
+
+    int transpose_buf_size = 2 * (8 * in_ch); /* to store intermediate data */
+    if (input_wd * input_ht < 8) {
+        transpose_buf_size = 0; // not using this for leftover
+    }
+    int align_buf_size = 32; /* extra buffer for alignment */
+    if (in_ch % 8 == 0 && filter_wd == 1 && filter_ht == 1 &&
+            pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) {
+        return filter_size + transpose_buf_size + align_buf_size;
+    }
+    return 2 * (filter_size + input_size) +  transpose_buf_size + align_buf_size;
+}
+
+void esp_nn_set_conv_scratch_buf_esp32s3(void *buf)
+{
+    scratch_buffer = (int16_t *) buf;
+}
+
+void esp_nn_conv_s8_esp32s3(const data_dims_t *input_dims,
+                            const int8_t *input,
+                            const data_dims_t *filter_dims,
+                            const int8_t *filter_data,
+                            const int32_t *bias,
+                            const data_dims_t *output_dims,
+                            int8_t *out_data,
+                            const conv_params_t *conv_params,
+                            const quant_data_t *quant_data)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t channels = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const uint16_t out_channels = output_dims->channels;
+    const int32_t *out_shift = quant_data->shift;
+    const int32_t *out_mult = quant_data->mult;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+
+    int filter_size = filter_wd * filter_ht * channels * out_channels;
+    int input_size = input_wd * input_ht * channels;
+    int align_len = 16 - (filter_size & 15);
+    int16_t *filter_data16 = scratch_buffer;
+    int16_t *input_data16 = scratch_buffer + filter_size + align_len;
+
+    if (scratch_buffer == NULL) {
+        printf("esp_nn_conv error! scratch_buffer not set!\n");
+        return;
+    }
+
+    if (channels % 8 == 0 && filter_wd == 1 && filter_ht == 1 &&
+            pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) {
+        int8_t *filter_aligned = (int8_t *) scratch_buffer;
+        int scratch_offset = (int) (filter_aligned + filter_size);
+        void *scratch_buf = (void *) (scratch_offset + 16 - (scratch_offset & 15));
+        memcpy(filter_aligned, filter_data, filter_size); // copy to aligned address
+        esp_nn_conv_s8_mult8_1x1_esp32s3(
+            input, input_wd, input_ht, channels, input_offset, filter_aligned,
+            bias, out_data, out_wd, out_ht, out_channels, out_offset,
+            out_shift, out_mult, activation_min, activation_max, scratch_buf);
+    } else if (channels % 4 == 0 && filter_wd == 1 && filter_ht == 1 &&
+            (input_wd * input_ht) % 4 == 0 && /* TODO: remove this check */
+            pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) {
+        int scratch_offset = (int) (input_data16 + input_size);
+        void *scratch_buf = (void *) (scratch_offset + 16 - (scratch_offset & 15));
+        esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
+        esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input, input_data16, input_size, input_offset);
+        esp_nn_conv_s16_mult4_1x1_esp32s3(
+            input_data16, input_wd, input_ht, channels, filter_data16,
+            bias, out_data, out_wd, out_ht, out_channels, out_offset,
+            out_shift, out_mult, activation_min, activation_max, scratch_buf);
+    } else if (channels % 8 == 0) {
+        esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
+        esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input, input_data16, input_size, input_offset);
+        esp_nn_conv_s16_mult8_esp32s3(
+            input_data16, input_wd, input_ht, channels, pad_wd, pad_ht,
+            stride_wd, stride_ht, filter_data16, filter_wd, filter_ht, bias,
+            out_data, out_wd, out_ht, out_channels, out_offset, out_shift,
+            out_mult, activation_min, activation_max);
+    } else if (pad_wd == 0 && pad_ht == 0) {
+        if (filter_wd == 3 && filter_ht == 3 && channels == 3) {
+            esp_nn_conv_s8_pad_valid_ch3_3x3(input, input_wd, input_ht, input_offset,
+                                             stride_wd, stride_ht, filter_data, bias,
+                                             out_data, out_wd, out_ht, out_channels, out_offset,
+                                             out_shift, out_mult, activation_min, activation_max);
+        } else {
+            esp_nn_conv_s8_pad_valid(input, input_wd, input_ht, channels, input_offset,
+                                     stride_wd, stride_ht, filter_data, filter_wd, filter_ht, bias,
+                                     out_data, out_wd, out_ht, out_channels, out_offset, out_shift,
+                                     out_mult, activation_min, activation_max);
+        }
+    } else {
+        /* Basic unrolled version */
+        esp_nn_conv_s8_unrolled(input_dims, input, filter_dims, filter_data,
+                                bias, output_dims, out_data, conv_params, quant_data);
+    }
+}

code/components/esp-nn/src/convolution/esp_nn_conv_opt.c

@@ -0,0 +1,179 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <esp_nn_defs.h>
+
+#include <common_functions.h>
+
+int esp_nn_get_conv_scratch_size_opt(const data_dims_t *input_dims,
+                                     const data_dims_t *filter_dims,
+                                     const data_dims_t *output_dims,
+                                     const conv_params_t *conv_params)
+{
+    return 0;
+}
+
+void esp_nn_set_conv_scratch_buf_opt(const void *buf)
+{
+
+}
+
+__attribute__ ((noinline))
+static void esp_nn_conv_s8_1x1(const data_dims_t *input_dims,
+                               const int8_t *input_data,
+                               const int8_t *filter_data,
+                               const int32_t *bias,
+                               const data_dims_t *output_dims,
+                               int8_t *out_data,
+                               const conv_params_t *conv_params,
+                               const quant_data_t *quant_data)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t in_channels = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const uint16_t out_channels = output_dims->channels;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+
+    for (int32_t in_row = 0; in_row < out_ht * stride_ht; in_row += stride_ht) {
+        for (int32_t in_col = 0; in_col < out_wd * stride_wd; in_col += stride_wd) {
+            const int32_t *out_mult = quant_data->mult;
+            const int32_t *out_shift = quant_data->shift;
+            const int8_t *filter_ptr = filter_data;
+            const int8_t *input_base_ptr = input_data + (in_row * input_wd + in_col) * in_channels;
+            int32_t out_ch_idx = 0;
+            for (; out_ch_idx < out_channels; out_ch_idx++) {
+                int32_t conv_out = 0;
+
+                const int8_t *input_ptr = input_base_ptr;
+
+                int32_t in_ch_idx = 0;
+                for (; in_ch_idx < in_channels - 3; in_ch_idx += 4) {
+                    conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                    conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                    conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                    conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                }
+                for (; in_ch_idx < in_channels; in_ch_idx ++) {
+                    conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                }
+                if (bias) {
+                    conv_out += bias[out_ch_idx];
+                }
+                conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, *out_mult++, *out_shift++);
+                conv_out += out_offset;
+                conv_out = max(conv_out, activation_min);
+                conv_out = min(conv_out, activation_max);
+                *out_data++ = (int8_t) conv_out;
+            }
+        }
+    }
+}
+
+/**
+ * Assumption 1: i/p channels == o/p channels
+ * Assumption 2: Pointers are valid
+ * Assumption 3: dialation width = 1
+ */
+void esp_nn_conv_s8_opt(const data_dims_t *input_dims,
+                        const int8_t *input_data,
+                        const data_dims_t *filter_dims,
+                        const int8_t *filter_data,
+                        const int32_t *bias,
+                        const data_dims_t *output_dims,
+                        int8_t *out_data,
+                        const conv_params_t *conv_params,
+                        const quant_data_t *quant_data)
+{
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+
+    if (filter_wd == 1 && filter_ht == 1) {
+        esp_nn_conv_s8_1x1(input_dims, input_data, filter_data, bias,
+                           output_dims, out_data, conv_params, quant_data);
+        return;
+    }
+
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t in_channels = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const uint16_t out_channels = output_dims->channels;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+
+    int32_t out_ch_idx, out_y, out_x, filter_y_idx, filter_x_idx;
+
+    for (out_y = 0; out_y < out_ht; out_y++) {
+        for (out_x = 0; out_x < out_wd; out_x++) {
+            const int32_t *out_shift = quant_data->shift;
+            const int32_t *out_mult = quant_data->mult;
+            for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
+                int32_t conv_out = 0;
+
+                const int32_t base_y = stride_ht * out_y - pad_ht;
+                const int32_t base_x = stride_wd * out_x - pad_wd;
+
+                const int32_t filter_y_start = max(0, -base_y);
+                const int32_t filter_x_start = max(0, -base_x);
+
+                const int32_t filter_y_end = min(filter_ht, input_ht - base_y);
+                const int32_t filter_x_end = min(filter_wd, input_wd - base_x);
+
+                for (filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                    for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                        const int32_t in_row = base_y + filter_y_idx;
+                        const int32_t in_col = base_x + filter_x_idx;
+
+                        const int8_t *input_ptr = input_data +
+                                        (in_row * input_wd + in_col) * in_channels;
+                        const int8_t *filter_ptr = filter_data +
+                                        out_ch_idx * in_channels * filter_ht * filter_wd +
+                                        (filter_y_idx * filter_wd + filter_x_idx) * in_channels;
+                        int32_t in_ch_idx = 0;
+                        for (; in_ch_idx < in_channels - 3; in_ch_idx += 4) {
+                            conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                            conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                            conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                            conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                        }
+                        for (; in_ch_idx < in_channels; in_ch_idx ++) {
+                            conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
+                        }
+                    }
+                }
+                if (bias) {
+                    conv_out += bias[out_ch_idx];
+                }
+                conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, *out_mult++, *out_shift++);
+                conv_out += out_offset;
+                conv_out = max(conv_out, activation_min);
+                conv_out = min(conv_out, activation_max);
+                *out_data++ = (int8_t) conv_out;
+            }
+        }
+    }
+}

code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_ansi.c

@@ -0,0 +1,100 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <esp_nn_defs.h>
+#include <common_functions.h>
+
+int esp_nn_get_depthwise_conv_scratch_size_ansi(const data_dims_t *input_dims,
+                                                const data_dims_t *filter_dims,
+                                                const data_dims_t *output_dims,
+                                                const dw_conv_params_t *conv_params)
+{
+    return 0;
+}
+
+void esp_nn_set_depthwise_conv_scratch_buf_ansi(const void *buf)
+{
+
+}
+
+void esp_nn_depthwise_conv_s8_ansi(const data_dims_t *input_dims,
+                                   const int8_t *input_data,
+                                   const data_dims_t *filter_dims,
+                                   const int8_t *filter_data,
+                                   const int32_t *bias,
+                                   const data_dims_t *output_dims,
+                                   int8_t *out_data,
+                                   const dw_conv_params_t *conv_params,
+                                   const quant_data_t *quant_data)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t channels = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const int32_t *out_shift = quant_data->shift;
+    const int32_t *out_mult = quant_data->mult;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+    const uint16_t ch_mult = conv_params->ch_mult;
+
+    int out_idx = 0;
+    for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
+        const int16_t base_y = (out_y * stride_ht) - pad_ht;
+        for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
+            const int16_t base_x = (out_x * stride_wd) - pad_wd;
+            for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop
+                for (int ch_mult_idx = 0; ch_mult_idx < ch_mult; ch_mult_idx++) {
+                    int32_t result = 0;
+                    const int out_ch_idx = ch_mult_idx + ch_idx * ch_mult;
+
+                    /* Select filter so as the point doesn't lie outside block */
+                    int filter_y_start = max(0, -base_y);
+                    int filter_x_start = max(0, -base_x);
+                    int filter_y_end = min(filter_ht, input_ht - base_y);
+                    int filter_x_end = min(filter_wd, input_wd - base_x);
+
+                    for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                        const int32_t idx_y = base_y + filter_y_idx;
+                        for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                            const int32_t idx_x = base_x + filter_x_idx;
+                            int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
+                            int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
+                            int32_t input_val = input_data[input_index] + input_offset;
+                            int32_t filter_val = filter_data[filter_index];
+                            result += input_val * filter_val;
+                        }
+                    }
+                    if (bias) {
+                        result += bias[out_ch_idx];
+                    }
+                    result = esp_nn_multiply_by_quantized_mult(result, out_mult[out_ch_idx], out_shift[out_ch_idx]);
+                    result += out_offset;
+                    result = max(result, activation_min);
+                    result = min(result, activation_max);
+
+                    out_data[out_idx++] = result;
+                }
+            }
+        }
+    }
+}

code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_opt.c

@@ -0,0 +1,291 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <esp_nn_defs.h>
+#include <common_functions.h>
+
+int esp_nn_get_depthwise_conv_scratch_size_opt(const data_dims_t *input_dims,
+                                               const data_dims_t *filter_dims,
+                                               const data_dims_t *output_dims,
+                                               const dw_conv_params_t *conv_params)
+{
+    return 0;
+}
+
+void esp_nn_set_depthwise_conv_scratch_buf_opt(const void *buf)
+{
+
+}
+
+/* common channel multiplier == 1 case */
+__attribute__ ((noinline))
+static void esp_nn_depthwise_conv_s8_ch_mult_1(const data_dims_t *input_dims,
+                                               const int8_t *input_data,
+                                               const data_dims_t *filter_dims,
+                                               const int8_t *filter_data,
+                                               const int32_t *bias,
+                                               const data_dims_t *output_dims,
+                                               int8_t *out_data,
+                                               const dw_conv_params_t *conv_params,
+                                               const quant_data_t *quant_data)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t channels = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+
+    int out_idx = 0;
+    for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
+        const int16_t base_y = (out_y * stride_ht) - pad_ht;
+        for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
+            const int16_t base_x = (out_x * stride_wd) - pad_wd;
+
+            const int32_t *out_shift = quant_data->shift;
+            const int32_t *out_mult = quant_data->mult;
+
+            /* Select filter so as the point doesn't lie outside block */
+            int filter_y_start = max(0, -base_y);
+            int filter_x_start = max(0, -base_x);
+            int filter_y_end = min(filter_ht, input_ht - base_y);
+            int filter_x_end = min(filter_wd, input_wd - base_x);
+
+            int ch_idx = 0;
+            for (; ch_idx < channels - 3; ch_idx += 4) {//channel_loop
+                int32_t result0 = 0;
+                int32_t result1 = 0;
+                int32_t result2 = 0;
+                int32_t result3 = 0;
+
+                for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                    const int32_t idx_y = base_y + filter_y_idx;
+                    for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                        const int32_t idx_x = base_x + filter_x_idx;
+                        int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
+                        int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels) + ch_idx;
+                        int32_t input_val0 = input_data[input_index + 0] + input_offset;
+                        int32_t input_val1 = input_data[input_index + 1] + input_offset;
+                        int32_t input_val2 = input_data[input_index + 2] + input_offset;
+                        int32_t input_val3 = input_data[input_index + 3] + input_offset;
+                        int32_t filter_val0 = filter_data[filter_index + 0];
+                        int32_t filter_val1 = filter_data[filter_index + 1];
+                        int32_t filter_val2 = filter_data[filter_index + 2];
+                        int32_t filter_val3 = filter_data[filter_index + 3];
+                        result0 += input_val0 * filter_val0;
+                        result1 += input_val1 * filter_val1;
+                        result2 += input_val2 * filter_val2;
+                        result3 += input_val3 * filter_val3;
+                    }
+                }
+                if (bias) {
+                    result0 += bias[ch_idx + 0];
+                    result1 += bias[ch_idx + 1];
+                    result2 += bias[ch_idx + 2];
+                    result3 += bias[ch_idx + 3];
+                }
+                result0 = esp_nn_multiply_by_quantized_mult_fast(result0, *out_mult++, *out_shift++);
+                result1 = esp_nn_multiply_by_quantized_mult_fast(result1, *out_mult++, *out_shift++);
+                result2 = esp_nn_multiply_by_quantized_mult_fast(result2, *out_mult++, *out_shift++);
+                result3 = esp_nn_multiply_by_quantized_mult_fast(result3, *out_mult++, *out_shift++);
+
+                result0 += out_offset;
+                result1 += out_offset;
+                result2 += out_offset;
+                result3 += out_offset;
+
+                result0 = max(result0, activation_min);
+                result1 = max(result1, activation_min);
+                result2 = max(result2, activation_min);
+                result3 = max(result3, activation_min);
+
+                result0 = min(result0, activation_max);
+                result1 = min(result1, activation_max);
+                result2 = min(result2, activation_max);
+                result3 = min(result3, activation_max);
+
+                out_data[out_idx++] = result0;
+                out_data[out_idx++] = result1;
+                out_data[out_idx++] = result2;
+                out_data[out_idx++] = result3;
+            }
+            for (; ch_idx < channels; ch_idx++) {//channel_loop
+                int32_t result = 0;
+
+                for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                    const int32_t idx_y = base_y + filter_y_idx;
+                    for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                        const int32_t idx_x = base_x + filter_x_idx;
+                        int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
+                        int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels) + ch_idx;
+                        int32_t input_val = input_data[input_index] + input_offset;
+                        int32_t filter_val = filter_data[filter_index];
+                        result += input_val * filter_val;
+                    }
+                }
+                if (bias) {
+                    result += bias[ch_idx];
+                }
+                result = esp_nn_multiply_by_quantized_mult_fast(result, *out_mult++, *out_shift++);
+                result += out_offset;
+                result = max(result, activation_min);
+                result = min(result, activation_max);
+
+                out_data[out_idx++] = result;
+            }
+        }
+    }
+}
+
+void esp_nn_depthwise_conv_s8_opt(const data_dims_t *input_dims,
+                                  const int8_t *input_data,
+                                  const data_dims_t *filter_dims,
+                                  const int8_t *filter_data,
+                                  const int32_t *bias,
+                                  const data_dims_t *output_dims,
+                                  int8_t *out_data,
+                                  const dw_conv_params_t *conv_params,
+                                  const quant_data_t *quant_data)
+{
+    const uint16_t ch_mult = conv_params->ch_mult;
+    if (ch_mult == 1) {
+        esp_nn_depthwise_conv_s8_ch_mult_1(input_dims, input_data, filter_dims, filter_data,
+                                           bias, output_dims, out_data, conv_params, quant_data);
+        return;
+    }
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t channels = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+
+    int out_idx = 0;
+    for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
+        const int16_t base_y = (out_y * stride_ht) - pad_ht;
+        for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
+            const int16_t base_x = (out_x * stride_wd) - pad_wd;
+
+            const int32_t *out_shift = quant_data->shift;
+            const int32_t *out_mult = quant_data->mult;
+
+            /* Select filter so as the point doesn't lie outside block */
+            int filter_y_start = max(0, -base_y);
+            int filter_x_start = max(0, -base_x);
+            int filter_y_end = min(filter_ht, input_ht - base_y);
+            int filter_x_end = min(filter_wd, input_wd - base_x);
+
+            for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop
+                int ch_mult_idx = 0;
+                for (; ch_mult_idx < ch_mult - 3; ch_mult_idx += 4) {
+                    int32_t result0 = 0;
+                    int32_t result1 = 0;
+                    int32_t result2 = 0;
+                    int32_t result3 = 0;
+                    const int out_ch_idx =  ch_idx * ch_mult + ch_mult_idx;
+
+                    for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                        const int32_t idx_y = base_y + filter_y_idx;
+                        for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                            const int32_t idx_x = base_x + filter_x_idx;
+                            int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
+                            int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
+                            int32_t input_val = input_data[input_index] + input_offset;
+                            int32_t filter_val0 = filter_data[filter_index + 0];
+                            int32_t filter_val1 = filter_data[filter_index + 1];
+                            int32_t filter_val2 = filter_data[filter_index + 2];
+                            int32_t filter_val3 = filter_data[filter_index + 3];
+                            result0 += input_val * filter_val0;
+                            result1 += input_val * filter_val1;
+                            result2 += input_val * filter_val2;
+                            result3 += input_val * filter_val3;
+                        }
+                    }
+                    if (bias) {
+                        result0 += bias[out_ch_idx + 0];
+                        result1 += bias[out_ch_idx + 1];
+                        result2 += bias[out_ch_idx + 2];
+                        result3 += bias[out_ch_idx + 3];
+                    }
+                    result0 = esp_nn_multiply_by_quantized_mult_fast(result0, *out_mult++, *out_shift++);
+                    result1 = esp_nn_multiply_by_quantized_mult_fast(result1, *out_mult++, *out_shift++);
+                    result2 = esp_nn_multiply_by_quantized_mult_fast(result2, *out_mult++, *out_shift++);
+                    result3 = esp_nn_multiply_by_quantized_mult_fast(result3, *out_mult++, *out_shift++);
+
+                    result0 += out_offset;
+                    result1 += out_offset;
+                    result2 += out_offset;
+                    result3 += out_offset;
+
+                    result0 = max(result0, activation_min);
+                    result1 = max(result1, activation_min);
+                    result2 = max(result2, activation_min);
+                    result3 = max(result3, activation_min);
+                    result0 = min(result0, activation_max);
+                    result1 = min(result1, activation_max);
+                    result2 = min(result2, activation_max);
+                    result3 = min(result3, activation_max);
+
+                    out_data[out_idx++] = result0;
+                    out_data[out_idx++] = result1;
+                    out_data[out_idx++] = result2;
+                    out_data[out_idx++] = result3;
+                }
+                for (; ch_mult_idx < ch_mult; ch_mult_idx++) {
+                    int32_t result = 0;
+                    const int out_ch_idx =  ch_idx * ch_mult + ch_mult_idx;
+
+                    for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                        const int32_t idx_y = base_y + filter_y_idx;
+                        for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                            const int32_t idx_x = base_x + filter_x_idx;
+                            int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
+                            int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
+                            int32_t input_val = input_data[input_index] + input_offset;
+                            int32_t filter_val = filter_data[filter_index];
+                            result += input_val * filter_val;
+                        }
+                    }
+                    if (bias) {
+                        result += bias[out_ch_idx];
+                    }
+                    result = esp_nn_multiply_by_quantized_mult_fast(result, *out_mult++, *out_shift++);
+                    result += out_offset;
+                    result = max(result, activation_min);
+                    result = min(result, activation_max);
+
+                    out_data[out_idx++] = result;
+                }
+            }
+        }
+    }
+}

code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_s8_esp32s3.c

@@ -0,0 +1,543 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+#include <esp_nn_defs.h>
+
+#include <common_functions.h>
+
+static int16_t *scratch_buffer = NULL;
+
+extern void esp_nn_depthwise_conv_s16_mult8_3x3_esp32s3(const int16_t *input_data,
+                                                        const uint16_t input_wd,
+                                                        const uint16_t input_ht,
+                                                        const uint16_t channels,
+                                                        const uint16_t pad_wd,
+                                                        const uint16_t pad_ht,
+                                                        const uint16_t stride_wd,
+                                                        const uint16_t stride_ht,
+                                                        const uint16_t ch_mult,
+                                                        const int16_t *filter_data,
+                                                        const int32_t *bias,
+                                                        int8_t *out_data,
+                                                        const uint16_t out_wd,
+                                                        const uint16_t out_ht,
+                                                        const int32_t out_offset,
+                                                        const int32_t *out_shift,
+                                                        const int32_t *out_mult,
+                                                        const int32_t activation_min,
+                                                        const int32_t activation_max);
+
+extern void esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(const int8_t *input_data,
+                                                              const uint16_t input_wd,
+                                                              const uint16_t input_ht,
+                                                              const uint16_t channels,
+                                                              const int32_t input_offset,
+                                                              const uint16_t stride_wd,
+                                                              const uint16_t stride_ht,
+                                                              const int8_t *filter_data,
+                                                              const int32_t *bias,
+                                                              int8_t *out_data,
+                                                              const uint16_t out_wd,
+                                                              const uint16_t out_ht,
+                                                              const int32_t out_offset,
+                                                              const int32_t *out_shift,
+                                                              const int32_t *out_mult,
+                                                              const int32_t activation_min,
+                                                              const int32_t activation_max);
+
+extern void esp_nn_depthwise_conv_s16_mult1_3x3_no_pad_esp32s3(const int16_t *input_data,
+                                                               const uint16_t input_wd,
+                                                               const uint16_t input_ht,
+                                                               const uint16_t channels,
+                                                               const uint16_t stride_wd,
+                                                               const uint16_t stride_ht,
+                                                               const int16_t *filter_data,
+                                                               const int32_t *bias,
+                                                               int8_t *out_data,
+                                                               const uint16_t out_wd,
+                                                               const uint16_t out_ht,
+                                                               const int32_t out_offset,
+                                                               const int32_t *out_shift,
+                                                               const int32_t *out_mult,
+                                                               const int32_t activation_min,
+                                                               const int32_t activation_max);
+
+extern void esp_nn_depthwise_conv_s16_mult8_esp32s3(const int16_t *input_data,
+                                                    const uint16_t input_wd,
+                                                    const uint16_t input_ht,
+                                                    const uint16_t channels,
+                                                    const uint16_t pad_wd,
+                                                    const uint16_t pad_ht,
+                                                    const uint16_t stride_wd,
+                                                    const uint16_t stride_ht,
+                                                    const uint16_t ch_mult,
+                                                    const int16_t *filter_data,
+                                                    const uint16_t filter_wd,
+                                                    const uint16_t filter_ht,
+                                                    const int32_t *bias,
+                                                    int8_t *out_data,
+                                                    const uint16_t out_wd,
+                                                    const uint16_t out_ht,
+                                                    const int32_t out_offset,
+                                                    const int32_t *out_shift,
+                                                    const int32_t *out_mult,
+                                                    const int32_t activation_min,
+                                                    const int32_t activation_max);
+
+extern void esp_nn_depthwise_conv_s16_mult4_esp32s3(const int16_t *input_data,
+                                                    const uint16_t input_wd,
+                                                    const uint16_t input_ht,
+                                                    const uint16_t channels,
+                                                    const uint16_t pad_wd,
+                                                    const uint16_t pad_ht,
+                                                    const uint16_t stride_wd,
+                                                    const uint16_t stride_ht,
+                                                    const uint16_t ch_mult,
+                                                    const int16_t *filter_data,
+                                                    const uint16_t filter_wd,
+                                                    const uint16_t filter_ht,
+                                                    const int32_t *bias,
+                                                    int8_t *out_data,
+                                                    const uint16_t out_wd,
+                                                    const uint16_t out_ht,
+                                                    const int32_t out_offset,
+                                                    const int32_t *out_shift,
+                                                    const int32_t *out_mult,
+                                                    const int32_t activation_min,
+                                                    const int32_t activation_max);
+
+extern void esp_nn_depthwise_conv_s16_mult1_3x3_esp32s3(const int16_t *input_data,
+                                                        const uint16_t input_wd,
+                                                        const uint16_t input_ht,
+                                                        const uint16_t channels,
+                                                        const uint16_t pad_wd,
+                                                        const uint16_t pad_ht,
+                                                        const uint16_t stride_wd,
+                                                        const uint16_t stride_ht,
+                                                        const int16_t *filter_data,
+                                                        const int32_t *bias,
+                                                        int8_t *out_data,
+                                                        const uint16_t out_wd,
+                                                        const uint16_t out_ht,
+                                                        const int32_t out_offset,
+                                                        const int32_t *out_shift,
+                                                        const int32_t *out_mult,
+                                                        const int32_t activation_min,
+                                                        const int32_t activation_max);
+
+extern void esp_nn_depthwise_conv_s16_mult1_esp32s3(const int16_t *input_data,
+                                                    const uint16_t input_wd,
+                                                    const uint16_t input_ht,
+                                                    const uint16_t channels,
+                                                    const uint16_t pad_wd,
+                                                    const uint16_t pad_ht,
+                                                    const uint16_t stride_wd,
+                                                    const uint16_t stride_ht,
+                                                    const int16_t *filter_data,
+                                                    const uint16_t filter_wd,
+                                                    const uint16_t filter_ht,
+                                                    const int32_t *bias,
+                                                    int8_t *out_data,
+                                                    const uint16_t out_wd,
+                                                    const uint16_t out_ht,
+                                                    const int32_t out_offset,
+                                                    const int32_t *out_shift,
+                                                    const int32_t *out_mult,
+                                                    const int32_t activation_min,
+                                                    const int32_t activation_max);
+
+extern void esp_nn_s8_to_s16_esp32s3(const int8_t *src, int16_t *dst, const int size);
+
+extern void esp_nn_aligned_s8_to_s16_with_offset_esp32s3(const int8_t *src, int16_t *dst,
+                                                         const int size, const int32_t offset);
+
+static void esp_nn_depthwise_conv_s8_unrolled(const int8_t *input_data,
+                                              const uint16_t input_wd,
+                                              const uint16_t input_ht,
+                                              const uint16_t channels,
+                                              const int32_t input_offset,
+                                              const uint16_t pad_wd,
+                                              const uint16_t pad_ht,
+                                              const uint16_t stride_wd,
+                                              const uint16_t stride_ht,
+                                              const uint16_t ch_mult,
+                                              const int8_t *filter_data,
+                                              const uint16_t filter_wd,
+                                              const uint16_t filter_ht,
+                                              const int32_t *bias,
+                                              int8_t *out_data,
+                                              const uint16_t out_wd,
+                                              const uint16_t out_ht,
+                                              const int32_t out_offset,
+                                              const int32_t *out_shift,
+                                              const int32_t *out_mult,
+                                              const int32_t activation_min,
+                                              const int32_t activation_max)
+{
+    int out_idx = 0;
+    for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
+        const int16_t base_y = (out_y * stride_ht) - pad_ht;
+        for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
+            const int16_t base_x = (out_x * stride_wd) - pad_wd;
+            for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop
+                int ch_mult_idx = 0;
+                for (; ch_mult_idx < ch_mult - 3; ch_mult_idx += 4) {
+                    int32_t result0 = 0, result1 = 0, result2 = 0, result3 = 0;
+                    const int out_ch_idx = ch_mult_idx + ch_idx * ch_mult;
+
+                    /* Select filter so as the point doesn't lie outside block */
+                    int filter_y_start = max(0, -base_y);
+                    int filter_x_start = max(0, -base_x);
+                    int filter_y_end = min(filter_ht, input_ht - base_y);
+                    int filter_x_end = min(filter_wd, input_wd - base_x);
+
+                    for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                        const int32_t idx_y = base_y + filter_y_idx;
+                        for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                            const int32_t idx_x = base_x + filter_x_idx;
+                            int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
+                            int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
+                            int32_t input_val = input_data[input_index] + input_offset;
+                            int32_t filter_val0 = filter_data[filter_index + 0];
+                            int32_t filter_val1 = filter_data[filter_index + 1];
+                            int32_t filter_val2 = filter_data[filter_index + 2];
+                            int32_t filter_val3 = filter_data[filter_index + 3];
+                            result0 += input_val * filter_val0;
+                            result1 += input_val * filter_val1;
+                            result2 += input_val * filter_val2;
+                            result3 += input_val * filter_val3;
+                        }
+                    }
+                    if (bias) {
+                        result0 += bias[out_ch_idx + 0];
+                        result1 += bias[out_ch_idx + 1];
+                        result2 += bias[out_ch_idx + 2];
+                        result3 += bias[out_ch_idx + 3];
+                    }
+                    result0 = esp_nn_multiply_by_quantized_mult(result0,
+                                out_mult[out_ch_idx + 0], out_shift[out_ch_idx + 0]);
+                    result1 = esp_nn_multiply_by_quantized_mult(result1,
+                                out_mult[out_ch_idx + 1], out_shift[out_ch_idx + 1]);
+                    result2 = esp_nn_multiply_by_quantized_mult(result2,
+                                out_mult[out_ch_idx + 2], out_shift[out_ch_idx + 2]);
+                    result3 = esp_nn_multiply_by_quantized_mult(result3,
+                                out_mult[out_ch_idx + 3], out_shift[out_ch_idx + 3]);
+
+                    result0 += out_offset;
+                    result1 += out_offset;
+                    result2 += out_offset;
+                    result3 += out_offset;
+
+                    result0 = max(result0, activation_min);
+                    result1 = max(result1, activation_min);
+                    result2 = max(result2, activation_min);
+                    result3 = max(result3, activation_min);
+
+                    result0 = min(result0, activation_max);
+                    result1 = min(result1, activation_max);
+                    result2 = min(result2, activation_max);
+                    result3 = min(result3, activation_max);
+
+                    out_data[out_idx++] = result0;
+                    out_data[out_idx++] = result1;
+                    out_data[out_idx++] = result2;
+                    out_data[out_idx++] = result3;
+                }
+
+                /* left-over */
+                for (; ch_mult_idx < ch_mult; ch_mult_idx++) {
+                    int32_t result = 0;
+                    const int out_ch_idx = ch_mult_idx + ch_idx * ch_mult;
+
+                    /* Select filter so as the point doesn't lie outside block */
+                    int filter_y_start = max(0, -base_y);
+                    int filter_x_start = max(0, -base_x);
+                    int filter_y_end = min(filter_ht, input_ht - base_y);
+                    int filter_x_end = min(filter_wd, input_wd - base_x);
+
+                    for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                        const int32_t idx_y = base_y + filter_y_idx;
+                        for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                            const int32_t idx_x = base_x + filter_x_idx;
+                            int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
+                            int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
+                            int32_t input_val = input_data[input_index] + input_offset;
+                            int32_t filter_val = filter_data[filter_index];
+                            result += input_val * filter_val;
+                        }
+                    }
+                    if (bias) {
+                        result += bias[out_ch_idx];
+                    }
+                    result = esp_nn_multiply_by_quantized_mult(result, out_mult[out_ch_idx], out_shift[out_ch_idx]);
+                    result += out_offset;
+                    result = max(result, activation_min);
+                    result = min(result, activation_max);
+
+                    out_data[out_idx++] = result;
+                }
+            }
+        }
+    }
+}
+
+void esp_nn_depthwise_conv_s8_ch_mult1(const int8_t *input_data,
+                                       const uint16_t input_wd,
+                                       const uint16_t input_ht,
+                                       const uint16_t channels,
+                                       const int32_t input_offset,
+                                       const uint16_t pad_wd,
+                                       const uint16_t pad_ht,
+                                       const uint16_t stride_wd,
+                                       const uint16_t stride_ht,
+                                       const int8_t *filter_data,
+                                       const uint16_t filter_wd,
+                                       const uint16_t filter_ht,
+                                       const int32_t *bias,
+                                       int8_t *out_data,
+                                       const uint16_t out_wd,
+                                       const uint16_t out_ht,
+                                       const int32_t out_offset,
+                                       const int32_t *out_shift,
+                                       const int32_t *out_mult,
+                                       const int32_t activation_min,
+                                       const int32_t activation_max)
+{
+    int out_idx = 0;
+    for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
+        const int16_t base_y = (out_y * stride_ht) - pad_ht;
+        for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
+            const int16_t base_x = (out_x * stride_wd) - pad_wd;
+            for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop
+                int32_t result = 0;
+                /* Select filter so as the point doesn't lie outside block */
+                int filter_y_start = max(0, -base_y);
+                int filter_x_start = max(0, -base_x);
+                int filter_y_end = min(filter_ht, input_ht - base_y);
+                int filter_x_end = min(filter_wd, input_wd - base_x);
+
+                for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
+                    const int32_t idx_y = base_y + filter_y_idx;
+                    for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
+                        const int32_t idx_x = base_x + filter_x_idx;
+                        int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
+                        int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * channels + ch_idx;
+                        int32_t input_val = input_data[input_index] + input_offset;
+                        int32_t filter_val = filter_data[filter_index];
+                        result += input_val * filter_val;
+                    }
+                }
+                if (bias) {
+                    result += bias[ch_idx];
+                }
+                result = esp_nn_multiply_by_quantized_mult(result, out_mult[ch_idx], out_shift[ch_idx]);
+                result += out_offset;
+                result = max(result, activation_min);
+                result = min(result, activation_max);
+
+                out_data[out_idx++] = result;
+            }
+        }
+    }
+}
+
+int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const data_dims_t *input_dims,
+                                                   const data_dims_t *filter_dims,
+                                                   const data_dims_t *output_dims,
+                                                   const dw_conv_params_t *conv_params)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t channels = input_dims->channels;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t ch_mult = conv_params->ch_mult;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+
+    int filter_size = filter_wd * filter_ht * channels * ch_mult;
+    int pad_width = 0, pad_height = 0;
+
+    if ((ch_mult == 1) && (channels % 8 == 0) && (filter_wd == 3) && (filter_ht == 3)) {
+        if (channels % 16 == 0) {
+            if (pad_wd || pad_ht) {
+                pad_width = pad_wd * 2;
+                pad_height = pad_ht * 2;
+            } else {
+                // check if we need to pad additionally
+                pad_width = (out_wd * stride_wd + filter_wd - 1) - input_wd;
+                pad_height = (out_ht * stride_ht + filter_ht - 1) - input_ht;
+                // printf("in(%d %d %d), out(%d %d), filter (%d %d) stride (%d %d), pad (%d %d)",
+                //         input_wd, input_ht, channels, out_wd, out_ht, filter_wd, filter_ht,
+                //         stride_wd, stride_ht, pad_wd, pad_ht);
+            }
+            if (pad_width || pad_height) {
+                int input_size = (input_wd + pad_width) * (input_ht + pad_height) * channels;
+                // printf("ask1 %d\n", filter_size + input_size + 16);
+                return filter_size + input_size + 16;  // 16 for alignment
+            } else {
+                // printf("ask2 %d\n", filter_size + 16);
+                return filter_size + 16;  // 16 for alignment
+            }
+        } else {
+            int input_size = input_wd * input_ht * channels;
+            // printf("ask3 %d\n", 2 * (filter_size + input_size) + 16);
+            return  2 * (filter_size + input_size) + 16; // 16 for alignment
+        }
+    } else if (ch_mult % 4 == 0) {
+        int input_size = input_wd * input_ht * channels;
+        // printf("ask4 %d\n", 2 * (filter_size + input_size) + 16);
+        return  2 * (filter_size + input_size) + 16; // 16 for alignment
+    }
+    return 32; // just few bytes
+}
+
+void esp_nn_set_depthwise_conv_scratch_buf_esp32s3(void *buf)
+{
+    scratch_buffer = (int16_t *) buf;
+}
+
+/**
+ * Assumption 1: i/p channels == o/p channels
+ * Assumption 2: Pointers are valid
+ * Assumption 3: dialation width = 1
+ */
+
+
+
+void esp_nn_depthwise_conv_s8_esp32s3(const data_dims_t *input_dims,
+                                      const int8_t *input_data,
+                                      const data_dims_t *filter_dims,
+                                      const int8_t *filter_data,
+                                      const int32_t *bias,
+                                      const data_dims_t *output_dims,
+                                      int8_t *out_data,
+                                      const dw_conv_params_t *conv_params,
+                                      const quant_data_t *quant_data)
+{
+    const uint16_t input_wd = input_dims->width;
+    const uint16_t input_ht = input_dims->height;
+    const uint16_t channels = input_dims->channels;
+    const int32_t input_offset = conv_params->in_offset;
+    const int32_t out_offset = conv_params->out_offset;
+    const uint16_t pad_wd = conv_params->padding.width;
+    const uint16_t pad_ht = conv_params->padding.height;
+    const uint16_t stride_wd = conv_params->stride.width;
+    const uint16_t stride_ht = conv_params->stride.height;
+    const uint16_t filter_wd = filter_dims->width;
+    const uint16_t filter_ht = filter_dims->height;
+    const uint16_t out_wd = output_dims->width;
+    const uint16_t out_ht = output_dims->height;
+    const int32_t *out_shift = quant_data->shift;
+    const int32_t *out_mult = quant_data->mult;
+    const int32_t activation_min = conv_params->activation.min;
+    const int32_t activation_max = conv_params->activation.max;
+    const uint16_t ch_mult = conv_params->ch_mult;
+
+    int filter_size = filter_wd * filter_ht * channels * ch_mult;
+    int align_len = 16 - (filter_size & 15);
+    int input_size = input_wd * input_ht * channels;
+    int16_t *filter_data16 = scratch_buffer;
+    int16_t *input_data16 = scratch_buffer + filter_size + align_len;
+    if (scratch_buffer == NULL) {
+        printf("esp_nn_depthwise_conv error! scratch_buffer not set!\n");
+        return;
+    }
+
+    if ((ch_mult == 1) && (channels % 8 == 0)) {
+        if ((filter_wd == 3) && (filter_ht == 3)) {
+            if ((channels % 16 == 0) && (pad_wd == 1) && (pad_ht == 1)) {
+                /* process in 8 bits */
+                int8_t *filter_aligned = (int8_t *) scratch_buffer;
+                int8_t *input_padded = (int8_t *) scratch_buffer + filter_size + align_len;
+                memcpy(filter_aligned, filter_data, filter_size);
+                esp_nn_aligned_s8_pad_with_value(input_data, input_padded, input_wd, input_ht, channels,
+                                                 -input_offset, pad_wd, pad_ht);
+                esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(input_padded, input_wd + 2 * pad_wd,
+                                                                  input_ht + 2 * pad_ht, channels, input_offset,
+                                                                  stride_wd, stride_ht, filter_aligned, bias,
+                                                                  out_data, out_wd, out_ht, out_offset, out_shift,
+                                                                  out_mult, activation_min, activation_max);
+            } else if ((channels % 16 == 0) && (pad_wd == 0) && (pad_ht == 0)) {
+                /* process in 8 bits */
+                int8_t *filter_aligned = (int8_t *) scratch_buffer;
+                int8_t *input_padded = (int8_t *) scratch_buffer + filter_size + align_len;
+
+                // check if we need to pad additionally
+                int pad_right = (out_wd * stride_wd + filter_wd - 1) - input_wd;
+                int pad_bottom = (out_ht * stride_ht + filter_ht - 1) - input_ht;
+                if (pad_right || pad_bottom) { // pad right and bottom
+                    esp_nn_aligned_s8_pad_end_with_value(input_data, input_padded, input_wd, input_ht,
+                                                         channels, -input_offset, pad_right, pad_bottom);
+                } else {
+                    input_padded = (int8_t *) input_data;
+                }
+                memcpy(filter_aligned, filter_data, filter_size);
+                esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(input_padded, input_wd + pad_right,
+                                                                  input_ht + pad_bottom, channels, input_offset,
+                                                                  stride_wd, stride_ht, filter_aligned, bias,
+                                                                  out_data, out_wd, out_ht, out_offset, out_shift,
+                                                                  out_mult, activation_min, activation_max);
+            } else { /* (channels % 8) == 0 */
+                esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
+                esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input_data, input_data16, input_size, input_offset);
+                esp_nn_depthwise_conv_s16_mult1_3x3_esp32s3(input_data16, input_wd, input_ht, channels,
+                                                            pad_wd, pad_ht, stride_wd, stride_ht, filter_data16,
+                                                            bias, out_data, out_wd, out_ht, out_offset, out_shift,
+                                                            out_mult, activation_min, activation_max);
+            }
+        } else { // all other ch_mult == 1, `channels % 8 == 0`
+            esp_nn_depthwise_conv_s8_ch_mult1(input_data, input_wd, input_ht, channels, input_offset,
+                                              pad_wd, pad_ht, stride_wd, stride_ht,
+                                              filter_data, filter_wd, filter_ht,
+                                              bias, out_data, out_wd, out_ht, out_offset, out_shift,
+                                              out_mult, activation_min, activation_max);
+        }
+    } else if (ch_mult % 8 == 0) {
+        esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
+        esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input_data, input_data16, input_size, input_offset);
+        if (filter_wd == 3 && filter_ht == 3) {
+            esp_nn_depthwise_conv_s16_mult8_3x3_esp32s3(input_data16, input_wd, input_ht, channels,
+                                                        pad_wd, pad_ht, stride_wd, stride_ht, ch_mult,
+                                                        filter_data16, bias,
+                                                        out_data, out_wd, out_ht, out_offset, out_shift,
+                                                        out_mult, activation_min, activation_max);
+        } else {
+            esp_nn_depthwise_conv_s16_mult8_esp32s3(input_data16, input_wd, input_ht, channels,
+                                                    pad_wd, pad_ht, stride_wd, stride_ht, ch_mult,
+                                                    filter_data16, filter_wd, filter_ht, bias,
+                                                    out_data, out_wd, out_ht, out_offset, out_shift,
+                                                    out_mult, activation_min, activation_max);
+        }
+    } else if (ch_mult % 4 == 0) {
+        esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
+        esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input_data, input_data16, input_size, input_offset);
+        esp_nn_depthwise_conv_s16_mult4_esp32s3(input_data16, input_wd, input_ht, channels,
+                                                pad_wd, pad_ht, stride_wd, stride_ht, ch_mult,
+                                                filter_data16, filter_wd, filter_ht, bias,
+                                                out_data, out_wd, out_ht, out_offset, out_shift,
+                                                out_mult, activation_min, activation_max);
+    } else {
+        esp_nn_depthwise_conv_s8_unrolled(input_data, input_wd, input_ht, channels, input_offset,
+                                          pad_wd, pad_ht, stride_wd, stride_ht, ch_mult,
+                                          filter_data, filter_wd, filter_ht,
+                                          bias, out_data, out_wd, out_ht, out_offset, out_shift,
+                                          out_mult, activation_min, activation_max);
+    }
+}

code/components/esp-nn/src/fully_connected/esp_nn_fully_connected_ansi.c

@@ -0,0 +1,50 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+
+#include <common_functions.h>
+
+void esp_nn_fully_connected_s8_ansi(const int8_t *input_data,
+                                    const int32_t input_offset,
+                                    const uint16_t row_len,
+                                    const int8_t *filter_data,
+                                    const int32_t filter_offset,
+                                    const int32_t *bias,
+                                    int8_t *out_data,
+                                    const uint16_t out_channels,
+                                    const int32_t out_offset,
+                                    const int32_t out_shift,
+                                    const int32_t out_mult,
+                                    const int32_t activation_min,
+                                    const int32_t activation_max)
+{
+    for (int32_t out_c = 0; out_c < out_channels; ++out_c) {
+        int32_t result = 0;
+        for (int32_t data_idx = 0; data_idx < row_len; data_idx++) {
+            int32_t filter_index = row_len * out_c + data_idx;
+            int32_t input_val = input_data[data_idx];
+            int32_t filter_val = filter_data[filter_index];
+            result += (filter_val + filter_offset) * (input_val + input_offset);
+        }
+        if (bias) {
+            result += bias[out_c];
+        }
+        result = esp_nn_multiply_by_quantized_mult(result, out_mult, out_shift);
+        result += out_offset;
+        result = max(result, activation_min);
+        result = min(result, activation_max);
+        out_data[out_c] = (int8_t) result;
+    }
+}

code/components/esp-nn/src/pooling/esp_nn_avg_pool_ansi.c

@@ -0,0 +1,72 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+
+#include <common_functions.h>
+
+void esp_nn_avg_pool_s8_ansi(const int8_t *input,
+                             const uint16_t input_wd,
+                             const uint16_t input_ht,
+                             int8_t *output,
+                             const uint16_t output_wd,
+                             const uint16_t output_ht,
+                             const uint16_t stride_wd,
+                             const uint16_t stride_ht,
+                             const uint16_t filter_wd,
+                             const uint16_t filter_ht,
+                             const uint16_t pad_wd,
+                             const uint16_t pad_ht,
+                             const int32_t activation_min,
+                             const int32_t activation_max,
+                             const uint16_t channels)
+{
+    int32_t base_y = -pad_ht;
+    for (int32_t out_y = 0; out_y < output_ht; out_y++, base_y += stride_ht) {
+        int32_t base_x = -pad_wd;
+        for (int32_t out_x = 0; out_x < output_wd; out_x++, base_x += stride_wd) {
+            for (int32_t ch_idx = 0; ch_idx < channels; ch_idx++) {
+                int32_t result = 0;
+                int32_t filter_cnt = 0;
+                /* Make sure filter does not cross the input box */
+                int32_t filter_y_start = max(0, -base_y);
+                int32_t filter_x_start = max(0, -base_x);
+
+                int32_t filter_y_end = min(filter_ht, input_ht - base_y);
+                int32_t filter_x_end = min(filter_wd, input_wd - base_x);
+
+                for (int32_t filter_y = filter_y_start; filter_y < filter_y_end; filter_y++) {
+                    for (int32_t filter_x = filter_x_start; filter_x < filter_x_end; filter_x++) {
+                        int32_t in_x_idx = base_x + filter_x;
+                        int32_t in_y_idx = base_y + filter_y;
+                        int32_t input_index = (in_y_idx * input_wd + in_x_idx) * channels + ch_idx;
+                        result += input[input_index];
+                        filter_cnt++;
+                    }
+                }
+
+                /* Rounded average */
+                result = result > 0 ? (result + filter_cnt / 2) / filter_cnt
+                                    : (result - filter_cnt / 2) / filter_cnt;
+
+                /* Activation function */
+                result = max(result, activation_min);
+                result = min(result, activation_max);
+
+                int32_t output_index = (out_y * output_wd + out_x) * channels + ch_idx;
+                output[output_index] = (int8_t) result;
+            }
+        }
+    }
+}

code/components/esp-nn/src/pooling/esp_nn_max_pool_ansi.c

@@ -0,0 +1,66 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+
+#include <common_functions.h>
+
+void esp_nn_max_pool_s8_ansi(const int8_t *input,
+                             const uint16_t input_wd,
+                             const uint16_t input_ht,
+                             int8_t *output,
+                             const uint16_t output_wd,
+                             const uint16_t output_ht,
+                             const uint16_t stride_wd,
+                             const uint16_t stride_ht,
+                             const uint16_t filter_wd,
+                             const uint16_t filter_ht,
+                             const uint16_t pad_wd,
+                             const uint16_t pad_ht,
+                             const int32_t activation_min,
+                             const int32_t activation_max,
+                             const uint16_t channels)
+{
+    int32_t base_y = -pad_ht;
+    for (int32_t out_y = 0; out_y < output_ht; out_y++, base_y += stride_ht) {
+        int32_t base_x = -pad_wd;
+        for (int32_t out_x = 0; out_x < output_wd; out_x++, base_x += stride_wd) {
+            /* Make sure filter does not cross the input box */
+            int32_t filter_y_start = max(0, -base_y);
+            int32_t filter_x_start = max(0, -base_x);
+            int32_t filter_y_end = min(filter_ht, input_ht - base_y);
+            int32_t filter_x_end = min(filter_wd, input_wd - base_x);
+
+            for (int32_t ch_idx = 0; ch_idx < channels; ch_idx++) {
+                int8_t result = INT8_MIN;
+
+                for (int32_t filter_y = filter_y_start; filter_y < filter_y_end; filter_y++) {
+                    for (int32_t filter_x = filter_x_start; filter_x < filter_x_end; filter_x++) {
+                        int32_t in_x_idx = base_x + filter_x;
+                        int32_t in_y_idx = base_y + filter_y;
+                        int32_t input_index = (in_y_idx * input_wd + in_x_idx) * channels + ch_idx;
+                        result = max(input[input_index], result);
+                    }
+                }
+
+                /* Activation function */
+                result = max(result, activation_min);
+                result = min(result, activation_max);
+
+                int32_t output_index = (out_y * output_wd + out_x) * channels + ch_idx;
+                output[output_index] = result;
+            }
+        }
+    }
+}

code/components/esp-nn/src/softmax/esp_nn_softmax_ansi.c

@@ -0,0 +1,88 @@
+// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "softmax_common.h"
+
+int32_t esp_nn_get_softmax_scratch_size_ansi(const int32_t width, const int32_t height)
+{
+    (void) width;
+    (void) height;
+    return 0;
+}
+
+void esp_nn_set_softmax_scratch_buf_ansi(void *buffer)
+{
+    (void) buffer;
+    return;
+}
+
+void esp_nn_softmax_s8_ansi(const int8_t *input_data,
+                            const int32_t height,
+                            const int32_t width,
+                            const int32_t mult,
+                            const int32_t shift,
+                            const int32_t diff_min,
+                            int8_t *output_data)
+{
+    // The representation chosen for the input to the exp() function is Q5.26.
+    // We need to leave extra space since values that we skip might be as large as
+    // -32 before multiplying by input mult, and therefore as large as
+    // -16 afterwards.  Note that exp(-8) is definitely not insignificant to
+    // accumulation, but exp(-16) definitely is.
+#define ACCUM_BITS  12
+#define DIFF_BITS   5
+
+    const int32_t mask = (1 << shift);
+    int32_t col = 0;
+    const int8_t *in_ptr = input_data;
+    int8_t *out_ptr = output_data;
+
+    for (int row_idx = 0; row_idx < height; row_idx++) {
+        int8_t max_in_row = in_ptr[0];
+        for (col = 1; col < width; col++) {
+            max_in_row = max(max_in_row, in_ptr[col]);
+        }
+
+        int32_t input_diff = 0;
+        int32_t sum_of_exps = 0;
+
+        for (col = 0; col < width; col++) {
+            input_diff = in_ptr[col] - max_in_row;
+            if (input_diff >= diff_min) {
+                const int32_t input_diff_rescaled = SAT_HIGH_MUL(input_diff * mask, mult);
+                const int32_t exp_raw = esp_nn_exp_on_negative_values(input_diff_rescaled);
+                sum_of_exps += DIV_POW2(exp_raw, ACCUM_BITS);
+            }
+        }
+
+        const int32_t headroom_plus1 = esp_nn_clz32((uint32_t) sum_of_exps);
+        const int32_t shifted_scale = ONE_OVER_ONE_X((sum_of_exps << headroom_plus1) - (1 << 31));
+        const int32_t bits_over_unit = ACCUM_BITS - headroom_plus1 + 31 - sizeof(int8_t) * 8;
+
+        for (col = 0; col < width; col++) {
+            input_diff = in_ptr[col] - max_in_row;
+            if (input_diff >= diff_min) {
+                const int32_t input_diff_rescaled = SAT_HIGH_MUL(input_diff * mask, mult);
+                const int32_t exp_raw = esp_nn_exp_on_negative_values(input_diff_rescaled);
+                const int32_t shifted_output = SAT_HIGH_MUL(shifted_scale, exp_raw);
+                const int32_t result = DIV_POW2(shifted_output, bits_over_unit) - 128;
+                out_ptr[col] = (int8_t) esp_nn_saturate8(result);
+            } else {
+                out_ptr[col] = -128;
+            }
+        }
+        in_ptr  += width;
+        out_ptr += width;
+    }
+}

code/components/esp-nn/src/softmax/esp_nn_softmax_opt.c

@@ -0,0 +1,108 @@
+// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "softmax_common.h"
+#include <stdio.h>
+
+static int32_t *scratch_buf = NULL;
+
+/**
+ * @brief   Get scratch buffer size needed by softmax function
+ *
+ * @param   width
+ * @param   height
+ * @return  size in bytes
+ *
+ * @note    buffer must be 4 byte aligned
+ */
+int32_t esp_nn_get_softmax_scratch_size_opt(const int32_t width, const int32_t height)
+{
+    (void) height;
+    return width * 4;
+}
+
+/**
+ * @brief   Set scratch buffer to be used by softmax function
+ *
+ * @param   buffer  this can be NULL if one needs to unset it
+ *                  must be aligned to 4 bytes
+ */
+void esp_nn_set_softmax_scratch_buf_opt(void *buffer)
+{
+    scratch_buf = (int32_t *) buffer;
+}
+
+void esp_nn_softmax_s8_opt(const int8_t *input_data,
+                           const int32_t height,
+                           const int32_t width,
+                           const int32_t mult,
+                           const int32_t shift,
+                           const int32_t diff_min,
+                           int8_t *output_data)
+{
+    if (scratch_buf == NULL) {
+        printf("%s error! scratch buffer not set\n", __FUNCTION__);
+        return;
+    }
+    // The representation chosen for the input to the exp() function is Q5.26.
+    // We need to leave extra space since values that we skip might be as large as
+    // -32 before multiplying by input mult, and therefore as large as
+    // -16 afterwards.  Note that exp(-8) is definitely not insignificant to
+    // accumulation, but exp(-16) definitely is.
+#define ACCUM_BITS  12
+#define DIFF_BITS   5
+
+    const int32_t mask = (1 << shift);
+    int32_t col = 0;
+    const int8_t *in_ptr = input_data;
+    int8_t *out_ptr = output_data;
+
+    for (int row_idx = 0; row_idx < height; row_idx++) {
+        int8_t max_in_row = in_ptr[0];
+        for (col = 1; col < width; col++) {
+            max_in_row = max(max_in_row, in_ptr[col]);
+        }
+
+        int32_t input_diff = 0;
+        int32_t sum_of_exps = 0;
+
+        for (col = 0; col < width; col++) {
+            input_diff = in_ptr[col] - max_in_row;
+            if (input_diff >= diff_min) {
+                const int32_t input_diff_rescaled = SAT_HIGH_MUL(input_diff * mask, mult);
+                const int32_t exp_raw = esp_nn_exp_on_negative_values(input_diff_rescaled);
+                scratch_buf[col] = exp_raw; // store to avoid duplicate calculation later
+                sum_of_exps += DIV_POW2(exp_raw, ACCUM_BITS);
+            }
+        }
+
+        const int32_t headroom_plus1 = esp_nn_clz32((uint32_t) sum_of_exps);
+        const int32_t shifted_scale = ONE_OVER_ONE_X((sum_of_exps << headroom_plus1) - (1 << 31));
+        const int32_t bits_over_unit = ACCUM_BITS - headroom_plus1 + 31 - sizeof(int8_t) * 8;
+
+        for (col = 0; col < width; col++) {
+            input_diff = in_ptr[col] - max_in_row;
+            if (input_diff >= diff_min) {
+                int32_t exp_raw = scratch_buf[col];
+                const int32_t shifted_output = SAT_HIGH_MUL(shifted_scale, exp_raw);
+                const int32_t result = DIV_POW2(shifted_output, bits_over_unit) - 128;
+                out_ptr[col] = (int8_t) esp_nn_saturate8(result);
+            } else {
+                out_ptr[col] = -128;
+            }
+        }
+        in_ptr  += width;
+        out_ptr += width;
+    }
+}

code/components/esp-nn/src/softmax/softmax_common.h

@@ -0,0 +1,104 @@
+// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <common_functions.h>
+
+#define MASK_IF_ZERO(x)                 (x) == 0 ? ~0 : 0
+#define MASK_IF_NON_ZERO(x)             (x) != 0 ? ~0 : 0
+#define SELECT_USING_MASK(mask, a, b)   ((mask) & (a)) ^ (~(mask) & (b))
+#define SAT_HIGH_MUL(x, y)              esp_nn_sat_round_doubling_high_mul((x), (y))
+#define DIV_POW2(x,y)                   esp_nn_div_by_power_of_two((x), (y))
+
+__NN_FORCE_INLINE__ int32_t mul_power_of_2(int val, int exp)
+{
+    const int32_t thresh = ((1 << (31 - exp)) - 1);
+    int32_t result = val << exp;
+    result = SELECT_USING_MASK(MASK_IF_NON_ZERO(val > thresh), INT32_MAX, result);
+    result = SELECT_USING_MASK(MASK_IF_NON_ZERO(val < -thresh), INT32_MIN, result);
+    return result;
+}
+
+/**
+ * @brief   Calculate `1 / (1 + x)` for x in [0, 1]
+ *
+ * @param   val     input value to calculate `1/(1+x)` for
+ * @return  `int32_t` result
+ * @note    Newton-Raphson division
+ *
+ *          https://en.wikipedia.org/wiki/Division_algorithm#Newton.E2.80.93Raphson_division
+ *          Refer to that page for the logic behind the 48/17 and 32/17 constants.
+ *          Pseudocode: https://en.wikipedia.org/wiki/Division_algorithm#Pseudocode
+ */
+__NN_FORCE_INLINE__ int32_t esp_nn_one_over_one_plus_x_for_x_in_0_1(int32_t val)
+{
+    const int64_t sum = (int64_t) val + INT32_MAX;
+    const int32_t half_denominator = (int32_t) ((sum + (sum >= 0 ? 1 : -1)) / 2L);
+    int32_t constant_48_over_17 = 1515870810;
+    int32_t constant_neg_32_over_17 = -1010580540;
+    int32_t x = constant_48_over_17 + SAT_HIGH_MUL(half_denominator, constant_neg_32_over_17);
+    const int32_t fixed_2_one = (1 << 29);
+
+    x += mul_power_of_2(SAT_HIGH_MUL(x, fixed_2_one - SAT_HIGH_MUL(half_denominator, x)), 2);
+    x += mul_power_of_2(SAT_HIGH_MUL(x, fixed_2_one - SAT_HIGH_MUL(half_denominator, x)), 2);
+    x += mul_power_of_2(SAT_HIGH_MUL(x, fixed_2_one - SAT_HIGH_MUL(half_denominator, x)), 2);
+
+    return mul_power_of_2(x, 1);
+}
+
+#define ONE_OVER_ONE_X(x)   esp_nn_one_over_one_plus_x_for_x_in_0_1((x))
+
+/**
+ * @brief   Return exp(x) for x < 0.
+ *
+ */
+__NN_FORCE_INLINE__ int32_t esp_nn_exp_on_negative_values(int32_t val)
+{
+    int32_t shift = 24;
+
+    const int32_t one_quarter = (1 << shift);
+    int32_t mask = one_quarter - 1;
+    const int32_t val_mod_minus_quarter = (val & mask) - one_quarter;
+    const int32_t remainder             = val_mod_minus_quarter - val;
+
+    // calculate exponent for x in [-1/4, 0) in `result`
+    const int32_t x                     = (val_mod_minus_quarter << 5) + (1 << 28);
+    const int32_t x2                    = SAT_HIGH_MUL(x, x);
+    const int32_t x3                    = SAT_HIGH_MUL(x2, x);
+    const int32_t x4                    = SAT_HIGH_MUL(x2, x2);
+    const int32_t one_over_3            = 715827883;
+    const int32_t one_over_8            = 1895147668;
+
+    const int32_t x4_over_4 = DIV_POW2(x4, 2);
+    const int32_t x4_over_4_plus_x3_over_6_plus_x2_over_2 = DIV_POW2(SAT_HIGH_MUL(x4_over_4 + x3, one_over_3) + x2, 1);
+    int32_t result = one_over_8 + SAT_HIGH_MUL(one_over_8, x + x4_over_4_plus_x3_over_6_plus_x2_over_2);
+
+#define SELECT_IF_NON_ZERO(x) {                                   \
+    mask   = MASK_IF_NON_ZERO(remainder & (1 << shift++));        \
+    result = SELECT_USING_MASK(mask, SAT_HIGH_MUL(result, x), result); \
+}
+
+    SELECT_IF_NON_ZERO(1672461947)
+    SELECT_IF_NON_ZERO(1302514674)
+    SELECT_IF_NON_ZERO(790015084)
+    SELECT_IF_NON_ZERO(290630308)
+    SELECT_IF_NON_ZERO(39332535)
+    SELECT_IF_NON_ZERO(720401)
+    SELECT_IF_NON_ZERO(242)
+
+#undef SELECT_IF_NON_ZERO
+
+    mask = MASK_IF_ZERO(val);
+    return SELECT_USING_MASK(mask, INT32_MAX, result);
+}

code/components/esp-nn/test_app/CMakeLists.txt

@@ -0,0 +1,9 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(EXTRA_COMPONENT_DIRS "../" "../tests/")
+set(IDF_EXCLUDE_COMPONENTS test test_app)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(test_app)

code/components/esp-nn/test_app/main/CMakeLists.txt

@@ -0,0 +1,7 @@
+
+set(COMPONENT_SRCS "main.c")
+set(COMPONENT_ADD_INCLUDEDIRS "")
+
+set(COMPONENT_PRIV_REQUIRES tests)
+
+register_component()

code/components/esp-nn/test_app/main/component.mk

@@ -0,0 +1,8 @@
+#
+# Main component makefile.
+#
+# This Makefile can be left empty. By default, it will take the sources in the 
+# src/ directory, compile them and link them into lib(subdirectory_name).a
+# in the build directory. This behaviour is entirely configurable,
+# please read the ESP-IDF documents if you need to do this.
+# 

code/components/esp-nn/test_app/main/main.c

@@ -0,0 +1,87 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <freertos/FreeRTOS.h>
+#include <freertos/task.h>
+#include <esp_log.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <test_functions.h>
+#include <esp_timer.h>
+
+static const char *TAG = "test_app";
+static uint32_t start_c, start_opt, total_c, total_opt;
+
+void profile_c_start()
+{
+    /* initiate profiling */
+    start_c = esp_cpu_get_ccount();
+}
+
+void profile_c_end()
+{
+    /* record profile number */
+    total_c = esp_cpu_get_ccount() - start_c;
+}
+
+void profile_opt_start()
+{
+    /* initiate profiling */
+    start_opt = esp_cpu_get_ccount();
+}
+
+void profile_opt_end()
+{
+    /* record profile number */
+    total_opt = esp_cpu_get_ccount() - start_opt;
+}
+
+void app_main()
+{
+    /* s8 tests */
+    ESP_LOGI(TAG, "Running s8 tests...");
+    esp_nn_add_elementwise_s8_test();
+    printf("add, c %u opt %u\n", total_c, total_opt);
+    esp_nn_mul_elementwise_s8_test();
+    printf("mul, c %u opt %u\n", total_c, total_opt);
+    esp_nn_depthwise_conv_s8_test();
+    printf("depthwise, c %u opt %u\n", total_c, total_opt);
+    esp_nn_conv_s8_test();
+    printf("conv2d, c %u opt %u\n", total_c, total_opt);
+
+    esp_nn_relu6_s8_test();
+    printf("relu, c %u opt %u\n", total_c, total_opt);
+    esp_nn_avg_pool_s8_test();
+    printf("avg_pool, c %u opt %u\n", total_c, total_opt);
+    esp_nn_max_pool_s8_test();
+    printf("max_pool, c %u opt %u\n", total_c, total_opt);
+    esp_nn_fully_connected_s8_test();
+    printf("fully_connected, c %u opt %u\n", total_c, total_opt);
+    esp_nn_softmax_s8_test();
+    printf("softmax, c %u opt %u\n", total_c, total_opt);
+    ESP_LOGI(TAG, "s8 tests done!\n");
+
+    /* u8 tests */
+    //ESP_LOGI(TAG, "Running u8 tests...");
+    //esp_nn_add_elementwise_u8_test();
+    //esp_nn_depthwise_conv_u8_test();
+    //esp_nn_conv_u8_test();
+    //esp_nn_avg_pool_u8_test();
+    //esp_nn_max_pool_u8_test();
+    //esp_nn_fully_connected_u8_test();
+    //ESP_LOGI(TAG, "u8 tests done!\n");
+}

code/components/esp-nn/test_app/sdkconfig.defaults

@@ -0,0 +1,5 @@
+
+#
+# esp-nn
+#
+CONFIG_NN_ESP32=y

code/components/esp-nn/test_app/sdkconfig.defaults.esp32s3

@@ -0,0 +1,8 @@
+# Default configurations for ESP32-S3
+
+CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32S3_SPIRAM_SUPPORT=y
+
+CONFIG_ESP32S3_DATA_CACHE_64KB=y
+CONFIG_ESP32S3_DATA_CACHE_8WAYS=y
+CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y

code/components/esp-nn/tests/CMakeLists.txt

@@ -0,0 +1,15 @@
+
+set(COMPONENT_ADD_INCLUDEDIRS ./include/)
+set(COMPONENT_SRCS "src/basic_math_test.c"
+                   "src/convolution_test.c"
+                   "src/fully_connected_test.c"
+                   "src/pooling_test.c"
+                   "src/relu_test.c"
+                   "src/softmax_test.c")
+
+set(COMPONENT_REQUIRES )
+set(COMPONENT_PRIV_REQUIRES esp-nn)
+
+register_component()
+
+target_compile_options(${COMPONENT_LIB} PRIVATE -Wno-unused-function)

code/components/esp-nn/tests/README.md

@@ -0,0 +1,4 @@
+# Tests for esp_nn library
+
+- Include these in your test framework and run the framework.
+- For IDF test please refer `test_app`

code/components/esp-nn/tests/component.mk

@@ -0,0 +1,5 @@
+#FIXME
+
+COMPONENT_ADD_INCLUDEDIRS := include/
+
+COMPONENT_SRCDIRS :=  src/

code/components/esp-nn/tests/include/test_functions.h

@@ -0,0 +1,48 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+/* int8_t ops tests */
+void esp_nn_add_elementwise_s8_test();
+void esp_nn_mul_elementwise_s8_test();
+
+void esp_nn_depthwise_conv_s8_test();
+void esp_nn_conv_s8_test();
+
+void esp_nn_avg_pool_s8_test();
+void esp_nn_max_pool_s8_test();
+
+void esp_nn_fully_connected_s8_test();
+
+void esp_nn_relu6_s8_test();
+
+void esp_nn_softmax_s8_test();
+
+/* uint8_t ops tests */
+void esp_nn_add_elementwise_u8_test();
+
+void esp_nn_depthwise_conv_u8_test();
+void esp_nn_conv_u8_test();
+
+void esp_nn_avg_pool_u8_test();
+void esp_nn_max_pool_u8_test();
+
+void esp_nn_fully_connected_u8_test();
+
+/* instructions test functions */
+void compare_instructions_test();
+void arith_instructions_test();
+void min_max_instructions_test();
+void bitwise_instructions_test();
+void load_store_instructions_test();

code/components/esp-nn/tests/include/test_utils.h

@@ -0,0 +1,87 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <common_functions.h>
+#include <stdio.h>
+
+/* mult value range */
+#define MULT_MAX    INT32_MAX
+#define MULT_MIN    0
+
+/* shift value range */
+#define SHIFT_MIN   -31
+#define SHIFT_MAX   30
+
+/**
+ * @brief callback function to run before C function
+ */
+void profile_c_start();
+
+/**
+ * @brief callback function to run after C function
+ */
+void profile_c_end();
+
+/**
+ * @brief callback function to run before optimized function
+ */
+void profile_opt_start();
+
+/**
+ * @brief callback function to run after optimized function
+ */
+void profile_opt_end();
+
+#define ANSI_COLOR_RED     "\x1b[31m"
+#define ANSI_COLOR_GREEN   "\x1b[32m"
+#define ANSI_COLOR_YELLOW  "\x1b[33m"
+#define ANSI_COLOR_BLUE    "\x1b[34m"
+#define ANSI_COLOR_MAGENTA "\x1b[35m"
+#define ANSI_COLOR_CYAN    "\x1b[36m"
+#define ANSI_COLOR_RESET   "\x1b[0m"
+
+#define CHECK_EQUAL(ARRAY1, ARRAY2, size) ({    \
+    bool res = true;                            \
+    for (int _i = 0; _i < size; _i++) {         \
+        if (ARRAY1[_i] != ARRAY2[_i]) {         \
+            res = false;                        \
+            break;                              \
+        }                                       \
+    }                                           \
+    res;                                        \
+})
+
+#define PRINT_ARRAY_INT(ARRAY, width, height) ({        \
+    int *_array = (int *) ARRAY;                        \
+    for (int _j = 0; _j < height; _j++) {               \
+        for (int _i = 0; _i < width; _i++) {            \
+            printf("%d\t", _array[width * _j + _i]);    \
+        }                                               \
+        printf("\n");                                   \
+    }                                                   \
+    printf("\n");                                       \
+})
+
+#define PRINT_ARRAY_HEX(ARRAY, width, height) ({        \
+    uint8_t *_array = (uint8_t *) ARRAY;                \
+    for (int _j = 0; _j < height; _j++) {               \
+        for (int _i = 0; _i < width; _i++) {            \
+            printf("%02x\t", _array[width * _j + _i]);  \
+        }                                               \
+        printf("\n");                                   \
+    }                                                   \
+    printf("\n");                                       \
+})

code/components/esp-nn/tests/src/basic_math_test.c

@@ -0,0 +1,355 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <malloc.h>
+
+#include <common_functions.h>
+#include <esp_nn.h>
+#include "test_utils.h"
+
+#if CONFIG_IDF_CMAKE
+#if (CONFIG_SPIRAM_SUPPORT && (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC))
+#define IDF_HEAP_CAPS 1
+#endif
+
+#if IDF_HEAP_CAPS
+#include "esp_heap_caps.h"
+#endif
+#endif
+
+void esp_nn_add_elementwise_s8_test()
+{
+    /* prepare data */
+    const int size = 1600 + 8 + 7; /* odd len to test leftover */
+    int8_t *input1;
+    int8_t *input2;
+    int8_t *out_data_c;
+    int8_t *out_data_opt;
+    int8_t *input1_orig = NULL;
+    int8_t *input2_orig = NULL;
+    int8_t *out_c_orig = NULL;
+    int8_t *out_opt_orig = NULL;
+    int32_t input1_offset = 34;
+    int32_t input2_offset = 35;
+    int32_t output_offset = 36;
+    int32_t input1_shift = -8; // right_shift amt always <= 0
+    int32_t input2_shift = -8; // right_shift amt always <= 0
+    int32_t output_shift = -9; // right_shift amt always <= 0
+    int32_t left_shift = 15; // always +ve
+    int32_t input1_mult = INT32_MAX;
+    int32_t input2_mult = INT32_MAX;
+    int32_t output_mult = INT32_MAX;
+    int32_t activation_min = -128;
+    int32_t activation_max = 127;
+
+    for (int itr = 0; itr < 10; itr++) {
+        switch (itr) {
+        case 0: // all zeros
+            input1_offset = 0;
+            input2_offset = 0;
+            output_offset = 0;
+            input1_mult = 0;
+            input2_mult = 0;
+            output_mult = 0;
+            input1_shift = 0;
+            input2_shift = 0;
+            output_shift = 0;
+            left_shift = 0;
+        break;
+        case 1: // hit min
+            input1_offset = -127;
+            input2_offset = -127;
+            output_offset = -128;
+            input1_mult = MULT_MIN;
+            input2_mult = MULT_MIN;
+            output_mult = MULT_MIN;
+            input1_shift = 0;
+            input2_shift = 0;
+            output_shift = 0;
+            left_shift = 0;
+        break;
+        case 2: // hit max
+            input1_offset = 128;
+            input2_offset = 128;
+            output_offset = -127;
+            input1_mult = MULT_MAX;
+            input2_mult = MULT_MAX;
+            output_mult = MULT_MAX;
+            input1_shift = SHIFT_MIN;
+            input2_shift = SHIFT_MIN;
+            output_shift = SHIFT_MIN;
+            left_shift = 30 - 8; // since input is 8 bits
+        break;
+        case 3: // hit extreme max
+            input1_offset = 128;
+            input2_offset = 128;
+            output_offset = -127;
+            input1_mult = MULT_MAX;
+            input2_mult = MULT_MAX;
+            output_mult = MULT_MAX;
+            input1_shift = 0;
+            input2_shift = 0;
+            output_shift = 0;
+            left_shift = 30 - 8; // -8 since input is 8 bit
+        break;
+        default:  // practical random input
+            input1_offset = rand() % 256 - 127; // range [-127, 128]
+            input2_offset = rand() % 256 - 127; // range [-127, 128]
+            output_offset = rand() % 256 - 128; // range [-128, 127]
+            input1_mult = MULT_MAX / 2 + rand() % INT16_MAX;
+            input2_mult = MULT_MAX / 2 + rand() % INT16_MAX;
+            output_mult = MULT_MAX / 2 + rand() % INT16_MAX;
+            input1_shift = -8 + rand() % 4;
+            input2_shift = -8 + rand() % 4;
+            output_shift = -8 + rand() % 4;
+            left_shift = rand() % 15;
+        }
+#if IDF_HEAP_CAPS
+        input1_orig = (int8_t *) heap_caps_malloc(size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        input2_orig = (int8_t *) heap_caps_malloc(size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        out_c_orig = (int8_t *) heap_caps_malloc(size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        out_opt_orig = (int8_t *) heap_caps_malloc(size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+
+        input1 = 16 + input1_orig - ((uint32_t) input1_orig & 0xf);
+        input2 = 16 + input2_orig - ((uint32_t) input2_orig & 0xf);
+        out_data_c = 16 + out_c_orig - ((uint32_t) out_c_orig & 0xf);
+        out_data_opt = 16 + out_opt_orig - ((uint32_t) out_opt_orig & 0xf);
+#else
+        input1 = memalign(16, size);
+        input2 = memalign(16, size);
+        out_data_c = memalign(16, size);
+        out_data_opt = memalign(16, size);
+
+        input1_orig = input1;
+        input2_orig = input2;
+        out_c_orig = out_data_c;
+        out_opt_orig = out_data_opt;
+#endif
+        if (input1_orig == NULL || input2_orig == NULL || out_c_orig == NULL ||
+                out_opt_orig == NULL) {
+            printf(ANSI_COLOR_RED"%s error allocating buffers\n"ANSI_COLOR_RESET, __FUNCTION__);
+            goto elementwise_add_test_cleanup;
+        }
+
+        for (int i = 0; i < size; ++i) {
+            input1[i] = rand() % 256 - 128;
+            input2[i] = rand() % 256 - 128;
+        }
+
+        if (itr == 0) {
+            /* enable profiler */
+            profile_c_start();
+        }
+        /* C function */
+        esp_nn_add_elementwise_s8_ansi(input1, input2, input1_offset, input2_offset,
+                                       input1_mult, input2_mult, input1_shift, input2_shift,
+                                       left_shift, out_data_c, output_offset, output_mult,
+                                       output_shift, activation_min, activation_max, size);
+
+        if (itr == 0) {
+            profile_c_end();
+            profile_opt_start();
+        }
+
+        /* Optimized function */
+        esp_nn_add_elementwise_s8(input1, input2, input1_offset, input2_offset,
+                                  input1_mult, input2_mult, input1_shift, input2_shift,
+                                  left_shift, out_data_opt, output_offset, output_mult,
+                                  output_shift, activation_min, activation_max, size);
+        if (itr == 0) {
+            /* disable profiler */
+            profile_opt_end();
+        }
+
+        bool ret = CHECK_EQUAL(out_data_c, out_data_opt, size);
+        if (ret == false) {
+            printf(ANSI_COLOR_RED"%s[%d] failed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+            printf("Output: \n");
+            PRINT_ARRAY_HEX(out_data_opt, size, 1);
+            printf("Expected: \n");
+            PRINT_ARRAY_HEX(out_data_c, size, 1);
+            printf("Input1:\n");
+            PRINT_ARRAY_HEX(input1, size, 1);
+            printf("Input2:\n");
+            PRINT_ARRAY_HEX(input2, size, 1);
+            printf("in1_shift %d, in2_shift %d, left_shift %d, out_shift %d\n",
+                   input1_shift, input2_shift, left_shift, output_shift);
+            printf("in1_mult %d, in2_mult %d, out_mult %d\n", input1_mult, input2_mult, output_mult);
+            goto elementwise_add_test_cleanup;
+        }
+        printf(ANSI_COLOR_GREEN"%s[%d] passed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+
+elementwise_add_test_cleanup:
+        if (input1_orig) {
+            free(input1_orig);
+        }
+        if (input2_orig) {
+            free(input2_orig);
+        }
+        if (out_c_orig) {
+            free(out_c_orig);
+        }
+        if (out_opt_orig) {
+            free(out_opt_orig);
+        }
+    }
+}
+
+void esp_nn_mul_elementwise_s8_test()
+{
+    /* prepare data */
+    const int size = 1600 + 8 + 7; /* odd len to test leftover */
+    int8_t *input1;
+    int8_t *input2;
+    int8_t *out_data_c;
+    int8_t *out_data_opt;
+    int32_t input1_offset = 34;
+    int32_t input2_offset = 35;
+    int32_t output_offset = 36;
+    int32_t output_shift = -7;
+    int32_t output_mult = MULT_MAX; // max out_mult
+    int32_t activation_min = -128;
+    int32_t activation_max = 127;
+    int8_t *input1_orig = NULL;
+    int8_t *input2_orig = NULL;
+    int8_t *out_c_orig = NULL;
+    int8_t *out_opt_orig = NULL;
+
+    for (int itr = 0; itr < 10; itr++) {
+        switch (itr) {
+        case 0: // all zeros
+            input1_offset = 0;
+            input2_offset = 0;
+            output_offset = 0;
+            output_mult = 0;
+            output_shift = 0;
+        break;
+        case 1: // hit min
+            input1_offset = -127;
+            input2_offset = -127;
+            output_offset = -128;
+            output_mult = MULT_MIN;
+            output_shift = 0;
+        break;
+        case 2: // hit max
+            input1_offset = 128;
+            input2_offset = 128;
+            output_offset = -127;
+            output_mult = MULT_MAX;
+            output_shift = SHIFT_MIN;
+        break;
+        case 3: // hit extreme max
+            input1_offset = 128;
+            input2_offset = 128;
+            output_offset = -127;
+            output_mult = MULT_MAX;
+            output_shift = 0;
+        break;
+        default:  // practical random input
+            input1_offset = rand() % 256 - 127; // range [-127, 128]
+            input2_offset = rand() % 256 - 127; // range [-127, 128]
+            output_offset = rand() % 256 - 128; // range [-128, 127]
+            output_mult = MULT_MAX / 2 + rand() % INT16_MAX;
+            output_shift = -8 + rand() % 4;
+        }
+
+#if IDF_HEAP_CAPS
+        input1_orig = (int8_t *) heap_caps_malloc(size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        input2_orig = (int8_t *) heap_caps_malloc(size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        out_c_orig = (int8_t *) heap_caps_malloc(size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        out_opt_orig = (int8_t *) heap_caps_malloc(size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+
+        input1 = 16 + input1_orig - ((uint32_t) input1_orig & 0xf);
+        input2 = 16 + input2_orig - ((uint32_t) input2_orig & 0xf);
+        out_data_c = 16 + out_c_orig - ((uint32_t) out_c_orig & 0xf);
+        out_data_opt = 16 + out_opt_orig - ((uint32_t) out_opt_orig & 0xf);
+#else
+        input1 = memalign(16, size);
+        input2 = memalign(16, size);
+        out_data_c = memalign(16, size);
+        out_data_opt = memalign(16, size);
+
+        input1_orig = input1;
+        input2_orig = input2;
+        out_c_orig = out_data_c;
+        out_opt_orig = out_data_opt;
+#endif
+        if (input1_orig == NULL || input2_orig == NULL || out_c_orig == NULL ||
+                out_opt_orig == NULL) {
+            printf(ANSI_COLOR_RED"%s error allocating buffers\n"ANSI_COLOR_RESET, __FUNCTION__);
+            goto elementwise_mult_test_cleanup;
+        }
+
+        for (int i = 0; i < size; ++i) {
+            input1[i] = rand() % 256 - 128;
+            input2[i] = rand() % 256 - 128;
+        }
+
+        if (itr == 0) {
+            /* enable profiler */
+            profile_c_start();
+        }
+        /* C function */
+        esp_nn_mul_elementwise_s8_ansi(input1, input2, input1_offset, input2_offset,
+                                       out_data_c, output_offset, output_mult, output_shift,
+                                       activation_min, activation_max, size);
+
+        if (itr == 0) {
+            profile_c_end();
+            profile_opt_start();
+        }
+        /* Optimized function */
+        esp_nn_mul_elementwise_s8(input1, input2, input1_offset, input2_offset,
+                                  out_data_opt, output_offset, output_mult, output_shift,
+                                  activation_min, activation_max, size);
+
+        if (itr == 0) {
+            /* disable profiler */
+            profile_opt_end();
+        }
+
+        bool ret = CHECK_EQUAL(out_data_c, out_data_opt, size);
+        if (ret == false) {
+            printf(ANSI_COLOR_RED"%s[%d] failed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+            printf("Output: \n");
+            PRINT_ARRAY_HEX(out_data_opt, size, 1);
+            printf("Expected: \n");
+            PRINT_ARRAY_HEX(out_data_c, size, 1);
+            printf("Input1:\n");
+            PRINT_ARRAY_HEX(input1, size, 1);
+            printf("Input2:\n");
+            PRINT_ARRAY_HEX(input2, size, 1);
+            goto elementwise_mult_test_cleanup;
+        }
+        printf(ANSI_COLOR_GREEN"%s[%d] passed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+
+elementwise_mult_test_cleanup:
+        if (input1_orig) {
+            free(input1_orig);
+        }
+        if (input2_orig) {
+            free(input2_orig);
+        }
+        if (out_c_orig) {
+            free(out_c_orig);
+        }
+        if (out_opt_orig) {
+            free(out_opt_orig);
+        }
+    }
+}

code/components/esp-nn/tests/src/convolution_test.c

@@ -0,0 +1,605 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <malloc.h>
+
+#include <esp_nn.h>
+#include "test_utils.h"
+
+#if CONFIG_IDF_CMAKE
+#if (CONFIG_SPIRAM_SUPPORT && (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC))
+#define IDF_HEAP_CAPS 1
+#endif
+#if IDF_HEAP_CAPS
+#include "esp_heap_caps.h"
+#endif
+#endif
+
+void esp_nn_depthwise_conv_s8_test()
+{
+    int8_t *input = NULL, *filter_data = NULL, *out_data_c = NULL, *out_data_opt = NULL;
+    int32_t *bias = NULL;
+    int32_t input_offset = 5; /* some number in [-128, 127] */
+    int32_t out_offset = 7;
+    int32_t activation_min = -125;
+    int32_t activation_max = 120;
+    void *scratch_buf = NULL;
+
+    /* independent variables */
+    int input_wd, input_ht, channels;
+    uint16_t filter_ht, filter_wd, ch_mult;
+    uint16_t pad_wd, pad_ht, stride_wd, stride_ht;
+
+    // run for 15 iterations
+    for (int itr = 0; itr < 15; itr++) {
+        /* prepare data */
+        switch (itr) {
+        case 0: // (ch_mult 1, (channels % 16) = 0), filter (3,3), pad (0,0)
+            input_wd = 18;
+            input_ht = 18;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 1;
+            channels = 16;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 1: // (ch_mult 1, (channels % 16) = 0), filter (3,3), pad (1,1)
+            input_wd = 10;
+            input_ht = 10;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 1;
+            channels = 16;
+            pad_wd = 1;
+            pad_ht = 1;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 2: // (ch_mult 1, (channels % 8) = 0), filter (3,3), pad (1,1)
+            input_wd = 10;
+            input_ht = 10;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 1;
+            channels = 24;
+            pad_wd = 1;
+            pad_ht = 1;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 3: // other filter sizes (ch_mult 1, (channels % 8) = 0)
+            input_wd = 10;
+            input_ht = 10;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 1;
+            channels = 24;
+            pad_wd = 1;
+            pad_ht = 1;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 4: // other filter sizes (ch_mult 8 = 0)
+            input_wd = 6;
+            input_ht = 6;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 8;
+            channels = 4;
+            pad_wd = 1;
+            pad_ht = 1;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 5: // other filter sizes (ch_mult 8 = 0)
+            input_wd = 12;
+            input_ht = 12;
+            filter_ht = 5;
+            filter_wd = 5;
+            ch_mult = 8;
+            channels = 4;
+            pad_wd = 1;
+            pad_ht = 1;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 6: // other filter sizes (ch_mult 4 = 0)
+            input_wd = 6;
+            input_ht = 6;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 4;
+            channels = 4;
+            pad_wd = 1;
+            pad_ht = 1;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 7: // (ch_mult 1, (channels % 16) = 0), filter (3,3), pad (0,0)  stride (2,2)
+            input_wd = 6;
+            input_ht = 6;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 1;
+            channels = 16;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 2;
+            stride_ht = 2;
+            break;
+        case 8: // same as case 7, with large parameters
+            input_wd = 58;
+            input_ht = 58;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 1;
+            channels = 128;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 2;
+            stride_ht = 2;
+            break;
+        case 9: // (ch_mult 1, (channels % 16) = 0), filter (3,3), pad (0,0)  stride (2,2)
+            input_wd = 6;
+            input_ht = 6;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 1;
+            channels = 16;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 2;
+            stride_ht = 2;
+            break;
+        default:
+            input_wd = 6;
+            input_ht = 6;
+            filter_ht = 3;
+            filter_wd = 3;
+            ch_mult = 1;
+            channels = 16;
+            stride_wd = rand() % 2 + 1;
+            stride_ht = stride_wd;
+            pad_wd = stride_wd == 1 ? 0 : rand() % 2;
+            pad_ht = pad_wd;
+            printf("stride(%d), pad (%d)\t", stride_wd, pad_wd);
+            break;
+        }
+
+        uint16_t out_wd = (input_wd - filter_wd + 1) / stride_wd;
+        uint16_t out_ht = (input_ht - filter_ht + 1) / stride_ht;
+        if (itr == 9) {
+            // expect the function to handle this gracefully
+            out_wd += 1;
+            out_ht += 1;
+        }
+        int in_size = input_wd * input_ht * channels;
+        int out_size = out_wd * out_ht * channels * ch_mult;
+        int filter_size = filter_wd * filter_ht * channels * ch_mult + 4;
+        int bias_size = channels * ch_mult + 1;
+        int32_t out_shift[channels * ch_mult];
+        int32_t out_mult[channels * ch_mult];
+
+#if IDF_HEAP_CAPS
+        int8_t *input_orig = (int8_t *) heap_caps_malloc(in_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        int8_t *out_c_orig = (int8_t *) heap_caps_malloc(out_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        int8_t *out_opt_orig = (int8_t *) heap_caps_malloc(out_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        filter_data = (int8_t *) heap_caps_malloc(filter_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        bias = (int32_t *) heap_caps_malloc(bias_size * 4, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+
+        input = 16 + input_orig - ((uint32_t) input_orig & 0xf);
+        out_data_c = 16 + out_c_orig - ((uint32_t) out_c_orig & 0xf);
+        out_data_opt = 16 + out_opt_orig - ((uint32_t) out_opt_orig & 0xf);
+#else
+        input = memalign(16, in_size + 16);
+        filter_data = memalign(16, filter_size);
+        out_data_c = memalign(16, out_size + 16);
+        out_data_opt = memalign(16, out_size + 16);
+        bias = memalign(16, bias_size * 4);
+        int8_t *input_orig = input;
+        int8_t *out_c_orig = out_data_c;
+        int8_t *out_opt_orig = out_data_opt;
+#endif
+        if (bias == NULL || input == NULL || filter_data == NULL ||
+                out_data_c == NULL || out_data_opt == NULL || bias == NULL) {
+            printf(ANSI_COLOR_RED"%s[%d] allocations failed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+            goto dc_s8_cleanup;
+        }
+
+        /* Generate input data */
+        for (int i = 0; i < in_size; ++i) {
+            input[i] = rand() % 128;
+        }
+
+        /* Generate filter data */
+        for (int i = 0; i < filter_size; ++i) {
+            filter_data[i] = rand() % 256 - 128;
+        }
+
+        /* Generate bias data */
+        for (int i = 0; i < channels * ch_mult; ++i) {
+            bias[i + 1] = rand() % INT16_MAX; //0th index left for unalignment
+            out_shift[i] = -8 + rand() % 3;
+            out_mult[i] = 0x7eb0e200 + rand() % 50;
+        }
+
+        data_dims_t input_dims = {.width = input_wd, .height = input_ht, .channels = channels, 1};
+        data_dims_t output_dims = {.width = out_wd, .height = out_ht, .channels = channels * ch_mult, 1};
+        data_dims_t filter_dims = {.width = filter_wd, .height = filter_ht, 0, 0};
+        dw_conv_params_t conv_params = {.in_offset = input_offset, .out_offset = out_offset, .ch_mult = ch_mult,
+                                        .stride = {stride_wd, stride_ht}, .padding = {pad_wd, pad_ht},
+                                        .dilation = {0, 0}, .activation = {activation_min, activation_max}};
+        quant_data_t quant_data = {.shift = out_shift, .mult = out_mult};
+
+        int scratch_buf_size = esp_nn_get_depthwise_conv_scratch_size(&input_dims, &filter_dims,
+                                                                      &output_dims, &conv_params);
+        if (scratch_buf_size > 0) {
+#if IDF_HEAP_CAPS
+            scratch_buf = heap_caps_malloc(scratch_buf_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+            int align_sz = 16 - (((int32_t) scratch_buf) & 0xf);
+#else
+            scratch_buf = memalign(16, scratch_buf_size);
+            int align_sz = 0;
+#endif
+            if (scratch_buf == NULL) {
+                printf(ANSI_COLOR_RED"%s[%d] scratch_buf alloc failed size %d\n"ANSI_COLOR_RESET,
+                       __FUNCTION__, itr, scratch_buf_size);
+                goto dc_s8_cleanup;
+            }
+            esp_nn_set_depthwise_conv_scratch_buf(scratch_buf + align_sz);
+        }
+        if (itr == 0) {
+            /* enable profiler */
+            profile_c_start();
+        }
+
+        /* C function */
+        esp_nn_depthwise_conv_s8_ansi(&input_dims, input, &filter_dims, filter_data + 4,
+                                      bias + 1, &output_dims, out_data_c, &conv_params, &quant_data);
+
+        if (itr == 0) {
+            profile_c_end();
+            profile_opt_start();
+        }
+
+        /* Optimized function */
+        esp_nn_depthwise_conv_s8(&input_dims, input, &filter_dims, filter_data + 4,
+                                 bias + 1, &output_dims, out_data_opt, &conv_params, &quant_data);
+
+        if (itr == 0) {
+            /* disable profiler */
+            profile_opt_end();
+        }
+
+        bool ret = CHECK_EQUAL(out_data_c, out_data_opt, out_size);
+        if (ret == false) {
+            printf(ANSI_COLOR_RED"%s[%d] failed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+            printf("Output: \n");
+            PRINT_ARRAY_HEX(out_data_opt, out_size / out_ht, out_ht);
+            printf("Expected: \n");
+            PRINT_ARRAY_HEX(out_data_c, out_size / out_ht, out_ht);
+            printf("Input:\n");
+            PRINT_ARRAY_HEX(input, in_size / input_ht, input_ht);
+            printf("Filter data:\n");
+            PRINT_ARRAY_HEX(filter_data + 4, (filter_size - 4) / filter_ht, filter_ht);
+            printf("bias data:\n");
+            PRINT_ARRAY_INT(bias + 1, ch_mult * channels, 1);
+            goto dc_s8_cleanup;
+        }
+        printf(ANSI_COLOR_GREEN"%s[%d] passed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+
+    dc_s8_cleanup:
+        if (input) {
+            free(input_orig);
+        }
+        if (filter_data) {
+            free(filter_data);
+        }
+        if (out_data_c) {
+            free(out_c_orig);
+        }
+        if (out_data_opt) {
+            free(out_opt_orig);
+        }
+        if (bias) {
+            free(bias);
+        }
+        if (scratch_buf) {
+            free(scratch_buf);
+        }
+    }
+}
+
+void esp_nn_conv_s8_test()
+{
+    const int32_t input_offset = 5; /* some number in [-128, 127] */
+    const int32_t activation_min = -125;
+    const int32_t activation_max = 122;
+    const int32_t out_offset = 3;
+
+    void *scratch_buf = NULL;
+    int8_t *input_orig;
+    int8_t *out_c_orig;
+    int8_t *out_opt_orig;
+    int8_t *filter_data;
+    int32_t *bias;
+
+    /* independent variable */
+    int in_wd, in_ht, in_channels, out_channels;
+    uint16_t filter_ht, filter_wd;
+    uint16_t pad_wd, pad_ht, stride_wd, stride_ht;
+
+    // run for 10 iterations
+    for (int itr = 0; itr < 10; itr++) {
+        switch (itr) {
+        case 0: // ch % 8 == 0 && filter (1,1), padding (0,0)
+            in_wd = 10;
+            in_ht = 10;
+            in_channels = 64;
+            out_channels = 64;
+            filter_ht = 1;
+            filter_wd = 1;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 1: // ch % 4 == 0 && (in_wd * in_ht) % 16 == 0
+            in_wd = 4;
+            in_ht = 4;
+            in_channels = 20;
+            out_channels = 8;
+            filter_ht = 1;
+            filter_wd = 1;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 2: // ch, filter (3x3x3)
+            in_wd = 10;
+            in_ht = 10;
+            in_channels = 3;
+            out_channels = 64;
+            filter_ht = 3;
+            filter_wd = 3;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 3: // remaining pad (0, 0)
+            in_wd = 10;
+            in_ht = 10;
+            in_channels = 3;
+            out_channels = 64;
+            filter_ht = 1;
+            filter_wd = 1;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 4: // unopt case
+            in_wd = 10;
+            in_ht = 10;
+            in_channels = 12;
+            out_channels = 64;
+            filter_ht = 3;
+            filter_wd = 3;
+            pad_wd = 1;
+            pad_ht = 1;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        case 5: // ch % 8 == 0 & stride (2,2)
+            in_wd = 16;
+            in_ht = 16;
+            in_channels = 16;
+            out_channels = 16;
+            filter_ht = 1;
+            filter_wd = 1;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 2;
+            stride_ht = 2;
+            break;
+        case 6: // ch % 8 == 0 && filter (1,1), padding (0,0)
+            in_wd = 2;
+            in_ht = 2;
+            in_channels = 8;
+            out_channels = 8;
+            filter_ht = 1;
+            filter_wd = 1;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        default: // ch % 8 == 0
+            in_wd = 8;
+            in_ht = 8;
+            in_channels = 16;
+            out_channels = 16;
+            filter_ht = 1;
+            filter_wd = 1;
+            pad_wd = 0;
+            pad_ht = 0;
+            stride_wd = 1;
+            stride_ht = 1;
+            break;
+        }
+
+        /* prepare data */
+        uint16_t out_wd = (in_wd - filter_wd + 1) / stride_wd;
+        uint16_t out_ht = (in_ht - filter_ht + 1) / stride_ht;
+
+        int in_size = in_wd * in_ht * in_channels;
+        int filter_size = filter_wd * filter_ht * in_channels * out_channels + 2;
+        int out_size = out_wd * out_ht * out_channels;
+
+#if IDF_HEAP_CAPS
+        input_orig = (int8_t *) heap_caps_malloc(in_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        out_c_orig = (int8_t *) heap_caps_malloc(out_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        out_opt_orig = (int8_t *) heap_caps_malloc(out_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        filter_data = (int8_t *) heap_caps_malloc(filter_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+        bias = (int32_t *) heap_caps_malloc(128 + sizeof (int32_t) * out_channels, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+
+        int8_t *input = 16 + input_orig - ((uint32_t) input_orig & 0xf);
+        int8_t *out_data_c = 16 + out_c_orig - ((uint32_t) out_c_orig & 0xf);
+        int8_t *out_data_opt = 16 + out_opt_orig - ((uint32_t) out_opt_orig & 0xf);
+#else
+        int8_t *input = memalign(16, in_size);
+        int8_t *out_data_c = memalign(16, out_size);
+        int8_t *out_data_opt = memalign(16, out_size);
+        filter_data = memalign(16, filter_size);
+        bias = calloc(1, 128 + sizeof (int32_t) * out_channels);
+        input_orig = input;
+        out_c_orig = out_data_c;
+        out_opt_orig = out_data_opt;
+#endif
+        int32_t *out_shift = calloc(1, 128 + sizeof (int32_t) * out_channels);
+        int32_t *out_mult = calloc(1, 128 + sizeof (int32_t) * out_channels);
+
+        if (input == NULL || filter_data == NULL ||
+                out_data_c == NULL || out_data_opt == NULL) {
+            printf(ANSI_COLOR_RED"%s allocations failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+            goto conv_s8_cleanup;
+        }
+
+        if (bias == NULL || out_shift == NULL || out_mult == NULL) {
+            printf(ANSI_COLOR_RED"%s allocations failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+            goto conv_s8_cleanup;
+        }
+
+        /* Generate input data between -128 -> +127 */
+        for (int i = 0; i < in_size; ++i) {
+            input[i] = rand() % 255 - 128;
+        }
+
+        /* Generate filter data between -128 -> +127 */
+        for (int i = 0; i < filter_size; ++i) {
+            filter_data[i] = rand() % 256 - 128;
+        }
+
+        /* Generate bias data */
+        for (int i = 0; i < out_channels; ++i) {
+            bias[i] = (int32_t)rand() % UINT16_MAX + UINT8_MAX;
+        }
+
+        /* Shift and multiplier */
+        for (int i = 0; i < out_channels; ++i) {
+            out_shift[i] = -10 + rand() % 2;
+            out_mult[i] = 0x7f67f4f8 + rand() % 50;
+        }
+
+        data_dims_t input_dims = {.width = in_wd, .height = in_ht, .channels = in_channels, 1};
+        data_dims_t output_dims = {.width = out_wd, .height = out_ht, .channels = out_channels, 1};
+        data_dims_t filter_dims = {.width = filter_wd, .height = filter_ht, 0, 0};
+        conv_params_t conv_params = {.in_offset = input_offset, .out_offset = out_offset,
+                                    .stride = {stride_wd, stride_ht}, .padding = {pad_wd, pad_ht},
+                                    .dilation = {0, 0}, .activation = {activation_min, activation_max}};
+        quant_data_t quant_data = {.shift = out_shift, .mult = out_mult};
+
+        int scratch_buf_size = esp_nn_get_conv_scratch_size(&input_dims, &filter_dims,
+                                                            &output_dims, &conv_params);
+        if (scratch_buf_size > 0) {
+#if IDF_HEAP_CAPS
+            void *scratch_buf = heap_caps_malloc(scratch_buf_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+            int align_sz = 16 - (((int32_t) scratch_buf) & 0xf);
+#else
+            void *scratch_buf = memalign(16, scratch_buf_size);
+            int align_sz = 0;
+#endif
+            if (scratch_buf == NULL) {
+                printf(ANSI_COLOR_RED"%s scratch_buf alloc failed size %d\n"ANSI_COLOR_RESET, __FUNCTION__, scratch_buf_size);
+                goto conv_s8_cleanup;
+            }
+            esp_nn_set_conv_scratch_buf(scratch_buf + align_sz);
+        }
+
+        if (itr == 0) {
+            /* enable profiler */
+            profile_c_start();
+        }
+
+        /* C function */
+        esp_nn_conv_s8_ansi(&input_dims, input, &filter_dims, filter_data + 2,
+                            bias, &output_dims, out_data_c, &conv_params, &quant_data);
+
+        if (itr == 0) {
+            profile_c_end();
+            profile_opt_start();
+        }
+
+        /* Optimized function */
+        esp_nn_conv_s8(&input_dims, input, &filter_dims, filter_data + 2,
+                       bias, &output_dims, out_data_opt, &conv_params, &quant_data);
+
+        if (itr == 0) {
+            /* disable profiler */
+            profile_opt_end();
+        }
+
+        bool ret = CHECK_EQUAL(out_data_c, out_data_opt, out_size);
+        if (ret == false) {
+            printf(ANSI_COLOR_RED"%s[%d] failed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+            printf("Output: \n");
+            PRINT_ARRAY_HEX(out_data_opt, out_size / out_ht, out_ht);
+            printf("Expected: \n");
+            PRINT_ARRAY_HEX(out_data_c, out_size / out_ht, out_ht);
+            printf("Input:\n");
+            PRINT_ARRAY_HEX(input, in_size / in_ht, in_ht);
+            printf("Filter data:\n");
+            PRINT_ARRAY_HEX(filter_data + 2, (filter_size - 2) / filter_ht, filter_ht);
+            printf("bias data:\n");
+            PRINT_ARRAY_INT(bias, out_channels, 1);
+            goto conv_s8_cleanup;
+        }
+        printf(ANSI_COLOR_GREEN"%s[%d] passed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+
+    conv_s8_cleanup:
+        if (input) {
+            free(input_orig);
+        }
+        if (filter_data) {
+            free(filter_data);
+        }
+        if (out_data_c) {
+            free(out_c_orig);
+        }
+        if (out_data_opt) {
+            free(out_opt_orig);
+        }
+        if (bias) {
+            free(bias);
+        }
+        if (out_shift) {
+            free(out_shift);
+        }
+        if (out_mult) {
+            free(out_mult);
+        }
+        if (scratch_buf) {
+            free(scratch_buf);
+        }
+    }
+}

code/components/esp-nn/tests/src/fully_connected_test.c

@@ -0,0 +1,111 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <esp_nn.h>
+#include "test_utils.h"
+
+
+void esp_nn_fully_connected_s8_test()
+{
+    /* prepare data */
+    static uint16_t row_len = 256 + 8 + 7; /* odd len to test unaligned+left-over */
+    static uint16_t out_channels = 3;
+    int8_t input[row_len];
+    int8_t filter_data[row_len * out_channels];
+    int8_t output_c[out_channels], output_opt[out_channels];
+    static int32_t activation_min = -128;
+    static int32_t activation_max = 127;
+    static int32_t input_offset = 0;
+    static int32_t filter_offset = 0;
+    int32_t out_shift = -10;
+    static int32_t out_offset = 127;
+    int32_t out_mult = 0x59e492c4;
+    for (int itr = 0; itr < 5; itr++) {
+        out_mult = INT32_MAX / row_len + rand() % INT16_MAX;
+        switch (itr) {
+        case 0:
+            out_shift = -10;
+            break;
+        case 1:
+            out_shift = SHIFT_MIN;
+            break;
+        case 2:
+            out_shift = SHIFT_MAX;
+            break;
+        case 3:
+            out_shift = 0;
+            break;
+        default:
+            out_shift = -10 + rand() % 5;
+            break;
+        }
+        if (itr == 0) {
+            out_shift = SHIFT_MAX;
+        }
+        /* Generate input and filter data */
+        for (int i = 0; i < row_len; ++i) {
+            input[i] = rand() % 256 - 128;
+        }
+        for (int i = 0; i < row_len * out_channels; ++i) {
+            filter_data[i] = rand() % 256 - 128;
+        }
+
+        if (itr == 0) {
+            /* enable profiler */
+            profile_c_start();
+        }
+
+        /* C function */
+        esp_nn_fully_connected_s8_ansi(input, input_offset, row_len, filter_data, filter_offset,
+                                    NULL, output_c, out_channels, out_offset, out_shift, out_mult,
+                                    activation_min, activation_max);
+
+        if (itr == 0) {
+            profile_c_end();
+            profile_opt_start();
+        }
+
+        /* Optimized function */
+        esp_nn_fully_connected_s8(input, input_offset, row_len, filter_data, filter_offset,
+                                NULL, output_opt, out_channels, out_offset, out_shift, out_mult,
+                                activation_min, activation_max);
+
+        if (itr == 0) {
+            /* disable profiler */
+            profile_opt_end();
+        }
+
+        bool ret = CHECK_EQUAL(output_c, output_opt, out_channels);
+        if (ret == false) {
+            printf(ANSI_COLOR_RED"%s[%d] failed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+            printf("Output: \n");
+            PRINT_ARRAY_HEX(output_opt, out_channels, 1);
+            printf("Expected: \n");
+            PRINT_ARRAY_HEX(output_c, out_channels, 1);
+            printf("Input:\n");
+            PRINT_ARRAY_HEX(input, row_len, 1);
+            printf("Filter data:\n");
+            PRINT_ARRAY_HEX(filter_data, row_len, out_channels);
+            printf("Out shift: %d\n", out_shift);
+            printf("Out mult: %x\n", out_mult);
+            return;
+        }
+        printf(ANSI_COLOR_GREEN"%s[%d] passed\n"ANSI_COLOR_RESET, __FUNCTION__, itr);
+    }
+}

code/components/esp-nn/tests/src/pooling_test.c

@@ -0,0 +1,184 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <malloc.h>
+
+#include <esp_nn.h>
+#include "test_utils.h"
+
+
+void esp_nn_avg_pool_s8_test()
+{
+    /* prepare data */
+    const uint16_t input_wd = 16;
+    const uint16_t input_ht = 16;
+    const uint16_t channels = 16; /* With TFLite example, I have seen it 256 */
+    const int size = input_wd * input_ht * channels;
+    int8_t *input, *output_c, *output_opt;
+    const int32_t activation_min = -128;
+    const int32_t activation_max = 127;
+    const uint16_t pad_wd = 1;
+    const uint16_t pad_ht = 1;
+    const uint16_t stride_wd = 1;
+    const uint16_t stride_ht = 1;
+    const uint16_t filter_ht = 3;
+    const uint16_t filter_wd = 3;
+    const uint16_t out_wd = input_wd / stride_wd;
+    const uint16_t out_ht = input_ht / stride_ht;
+    const int out_size = out_wd * out_ht * channels;
+
+    input = memalign(16, size);
+    output_c = memalign(16, out_size);
+    output_opt = memalign(16, out_size);
+
+    if (input == NULL || output_c == NULL || output_opt == NULL) {
+        printf(ANSI_COLOR_RED"%s allocations failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+        goto avg_pool_s8_cleanup;
+    }
+    /**
+     * width/height, channels etc look suspicious but it it true.
+     * It actually depends upon where in model this is actually placed.
+     * If at the end wd/ht tends to be smaller and depth larger.
+     */
+
+    for (int i = 0; i < size; ++i) {
+        input[i] = rand() % 256 - 128;
+    }
+
+    /* enable profiler */
+    profile_c_start();
+
+    /* C function */
+    esp_nn_avg_pool_s8_ansi(input, input_wd, input_ht, output_c, out_wd, out_ht,
+                              stride_wd, stride_ht, filter_wd, filter_ht, pad_wd, pad_ht,
+                              activation_min, activation_max, channels);
+
+    profile_c_end();
+    profile_opt_start();
+
+    /* Optimized function */
+    esp_nn_avg_pool_s8(input, input_wd, input_ht, output_opt, out_wd, out_ht,
+                         stride_wd, stride_ht, filter_wd, filter_ht, pad_wd, pad_ht,
+                         activation_min, activation_max, channels);
+
+    /* disable profiler */
+    profile_opt_end();
+
+
+    bool ret = CHECK_EQUAL(output_c, output_opt, out_size);
+    if (ret == false) {
+        printf(ANSI_COLOR_RED"%s failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+        printf("Output: \n");
+        PRINT_ARRAY_HEX(output_opt, out_wd * channels, out_ht);
+        printf("Expected: \n");
+        PRINT_ARRAY_HEX(output_c, out_wd * channels, out_ht);
+        printf("Input:\n");
+        PRINT_ARRAY_HEX(input, input_wd * channels, input_ht);
+        goto avg_pool_s8_cleanup;
+    }
+    printf(ANSI_COLOR_GREEN"%s passed\n"ANSI_COLOR_RESET, __FUNCTION__);
+
+avg_pool_s8_cleanup:
+    if (input) {
+        free(input);
+    }
+    if (output_c) {
+        free(output_c);
+    }
+    if (output_opt) {
+        free(output_opt);
+    }
+}
+
+void esp_nn_max_pool_s8_test()
+{
+    /* prepare data */
+    const uint16_t input_wd = 16;
+    const uint16_t input_ht = 16;
+    const uint16_t channels = 16; /* With TFLite example, I have seen it 256 */
+    int8_t *input, *output_c, *output_opt;
+    const int size = input_wd * input_ht * channels;
+    const int32_t activation_min = -128;
+    const int32_t activation_max = 127;
+    const uint16_t pad_wd = 1;
+    const uint16_t pad_ht = 1;
+    const uint16_t stride_wd = 1;
+    const uint16_t stride_ht = 1;
+    const uint16_t filter_ht = 3;
+    const uint16_t filter_wd = 3;
+    const uint16_t out_wd = input_wd / stride_wd;
+    const uint16_t out_ht = input_ht / stride_ht;
+    const int out_size = out_wd * out_ht * channels;
+
+    input = memalign(16, size);
+    output_c = memalign(16, out_size);
+    output_opt = memalign(16, out_size);
+
+    if (input == NULL || output_c == NULL || output_opt == NULL) {
+        printf(ANSI_COLOR_RED"%s allocations failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+        goto max_pool_s8_cleanup;
+    }
+
+    for (int i = 0; i < size; ++i) {
+        input[i] = rand() % 256 - 128;
+    }
+
+    /* enable profiler */
+    profile_c_start();
+
+    /* C function */
+    esp_nn_max_pool_s8_ansi(input, input_wd, input_ht, output_c, out_wd, out_ht,
+                            stride_wd, stride_ht, filter_wd, filter_ht, pad_wd, pad_ht,
+                            activation_min, activation_max, channels);
+
+    profile_c_end();
+    profile_opt_start();
+
+    /* Optimized function */
+    esp_nn_max_pool_s8(input, input_wd, input_ht, output_opt, out_wd, out_ht,
+                       stride_wd, stride_ht, filter_wd, filter_ht, pad_wd, pad_ht,
+                       activation_min, activation_max, channels);
+
+    /* disable profiler */
+    profile_opt_end();
+
+
+    bool ret = CHECK_EQUAL(output_c, output_opt, out_wd * out_ht * channels);
+    if (ret == false) {
+        printf(ANSI_COLOR_RED"%s failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+        printf("Output: \n");
+        PRINT_ARRAY_HEX(output_opt, out_wd * out_ht * channels, 1);
+        printf("Expected: \n");
+        PRINT_ARRAY_HEX(output_c, out_wd * out_ht * channels, 1);
+        printf("Input:\n");
+        PRINT_ARRAY_HEX(input, 8, size / 8);
+        goto max_pool_s8_cleanup;
+    }
+    printf(ANSI_COLOR_GREEN"%s passed\n"ANSI_COLOR_RESET, __FUNCTION__);
+
+max_pool_s8_cleanup:
+    if (input) {
+        free(input);
+    }
+    if (output_c) {
+        free(output_c);
+    }
+    if (output_opt) {
+        free(output_opt);
+    }
+}

code/components/esp-nn/tests/src/relu_test.c

@@ -0,0 +1,83 @@
+// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <malloc.h>
+
+#include <esp_nn.h>
+#include "test_utils.h"
+
+void esp_nn_relu6_s8_test()
+{
+    const int size = 1600 + 8 + 7;
+    int8_t *input, *inout_ansi, *inout_opt;
+
+    input = memalign(16, size);
+    inout_ansi = memalign(16, size);
+    inout_opt = memalign(16, size);
+
+    if (input == NULL || inout_ansi == NULL || inout_opt == NULL) {
+        printf(ANSI_COLOR_RED"%s allocations failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+        goto relu6_s8_cleanup;
+    }
+    /* Generate filter data between -128 -> +127 */
+    for (int i = 0; i < size; ++i) {
+        input[i] = rand() % 255 - 128;
+        inout_ansi[i] = input[i];
+        inout_opt[i] = input[i];
+    }
+
+    /* enable profiler */
+    profile_c_start();
+
+    /* C function */
+    esp_nn_relu6_s8_ansi(inout_ansi, size);
+
+    profile_c_end();
+    profile_opt_start();
+
+    /* Optimized function */
+    esp_nn_relu6_s8(inout_opt, size);
+
+    /* disable profiler */
+    profile_opt_end();
+
+    bool ret = CHECK_EQUAL(inout_ansi, inout_opt, size);
+    if (ret == false) {
+        printf(ANSI_COLOR_RED"%s failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+        printf("Output: \n");
+        PRINT_ARRAY_HEX(inout_opt, size, 1);
+        printf("Expected: \n");
+        PRINT_ARRAY_HEX(inout_ansi, size, 1);
+        printf("Input:\n");
+        PRINT_ARRAY_HEX(input, size, 1);
+        goto relu6_s8_cleanup;
+    }
+    printf(ANSI_COLOR_GREEN"%s passed\n"ANSI_COLOR_RESET, __FUNCTION__);
+
+relu6_s8_cleanup:
+    if (input) {
+        free (input);
+    }
+    if (inout_ansi) {
+        free (inout_ansi);
+    }
+    if (inout_opt) {
+        free (inout_opt);
+    }
+
+}

code/components/esp-nn/tests/src/softmax_test.c

@@ -0,0 +1,101 @@
+// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <malloc.h>
+
+#include <esp_nn.h>
+#include "test_utils.h"
+
+void esp_nn_softmax_s8_test()
+{
+    const int32_t height = 8;
+    const int32_t width = 32;
+    const int32_t diff_min = -128;
+    const int32_t mult = INT32_MAX / 2;
+    const int32_t shift = 7;
+    void *scratch_buf = NULL;
+    const int size = width * height;
+    int8_t *input, *out_ansi, *out_opt;
+
+    input = memalign(16, size);
+    out_ansi = memalign(16, size);
+    out_opt = memalign(16, size);
+
+    if (input == NULL || out_ansi == NULL || out_opt == NULL) {
+        printf(ANSI_COLOR_RED"%s buffer allocations failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+        goto softmax_s8_cleanup;
+    }
+
+    /* Generate input data between -128 -> +127 */
+    for (int i = 0; i < size; ++i) {
+        input[i] = rand() % 255 - 128;
+    }
+
+    /* enable profiler */
+    profile_c_start();
+
+    /* C function */
+    esp_nn_softmax_s8_ansi(input, height, width, mult, shift, diff_min, out_ansi);
+
+    profile_c_end();
+
+    int32_t scratch_buf_size = esp_nn_get_softmax_scratch_size(width, height);
+    if (scratch_buf_size) {
+        scratch_buf = memalign(4, scratch_buf_size);
+        if (scratch_buf == NULL) {
+            printf(ANSI_COLOR_RED"%s scratch_buf alloc failed size %d\n"ANSI_COLOR_RESET, __FUNCTION__, scratch_buf_size);
+            goto softmax_s8_cleanup;
+        }
+        esp_nn_set_softmax_scratch_buf(scratch_buf);
+    }
+
+    profile_opt_start();
+
+    /* Optimized function */
+    esp_nn_softmax_s8(input, height, width, mult, shift, diff_min, out_opt);
+
+    /* disable profiler */
+    profile_opt_end();
+
+    bool ret = CHECK_EQUAL(out_ansi, out_opt, size);
+    if (ret == false) {
+        printf(ANSI_COLOR_RED"%s failed\n"ANSI_COLOR_RESET, __FUNCTION__);
+        printf("Output: \n");
+        PRINT_ARRAY_HEX(out_opt, width, height);
+        printf("Expected: \n");
+        PRINT_ARRAY_HEX(out_ansi, width, height);
+        printf("Input:\n");
+        PRINT_ARRAY_HEX(input, width, height);
+        goto softmax_s8_cleanup;
+    }
+    printf(ANSI_COLOR_GREEN"%s passed\n"ANSI_COLOR_RESET, __FUNCTION__);
+
+softmax_s8_cleanup:
+    if (input) {
+        free (input);
+    }
+    if (out_ansi) {
+        free (out_ansi);
+    }
+    if (out_opt) {
+        free (out_opt);
+    }
+    if (scratch_buf) {
+        free (scratch_buf);
+    }
+}

code/components/esp32-camera-master/.github/workflows/build.yml

@@ -0,0 +1,79 @@
+name: Build examples
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+
+jobs:
+  build-master:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+    - name: esp-idf build
+      uses: espressif/esp-idf-ci-action@latest
+      with:
+        path: 'examples'
+
+  build-release-v4_0:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+    - name: esp-idf build
+      uses: espressif/esp-idf-ci-action@release-v4.0
+      with:
+        path: 'examples'
+
+  build-release-v4_1:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+    - name: esp-idf build
+      uses: espressif/esp-idf-ci-action@release-v4.1
+      with:
+        path: 'examples'
+
+  build-release-v4_2:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+    - name: esp-idf build
+      uses: espressif/esp-idf-ci-action@release-v4.2
+      with:
+        path: 'examples'
+
+  build-release-v4_3:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+    - name: esp-idf build
+      uses: espressif/esp-idf-ci-action@release-v4.3
+      with:
+        path: 'examples'
+
+  build-release-v3_3:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+    - name: esp-idf build
+      uses: espressif/esp-idf-ci-action@release-v3.3
+      with:
+        path: 'examples'

code/components/esp32-camera-master/.github/workflows/stale.yml

@@ -0,0 +1,27 @@
+# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
+#
+# You can adjust the behavior by modifying this file.
+# For more information, see:
+# https://github.com/actions/stale
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: '20 9 * * *'
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue appears to be stale. Please close it if its no longer valid.'
+        stale-pr-message: 'This pull request appears to be stale. Please close it if its no longer valid.'
+        stale-issue-label: 'no-issue-activity'
+        stale-pr-label: 'no-pr-activity'

code/components/esp32-camera-master/.github/workflows/upload_component.yml

@@ -0,0 +1,21 @@
+name: Push component to https://components.espressif.com
+on:
+  push:
+    tags:
+      - v*
+jobs:
+  upload_components:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          submodules: "recursive"
+
+      - name: Upload component to the component registry
+        uses: espressif/github-actions/upload_components@master
+        with:
+          name: "esp32-camera"
+          version: "git"
+          namespace: "espressif"
+          service_url: ${{ secrets.IDF_COMPONENT_API_URL }}
+          api_token: ${{ secrets.IDF_COMPONENT_API_TOKEN }}

code/components/esp32-camera-master/.gitignore

@@ -1 +1,5 @@
 *.DS_Store
+.vscode
+**/build
+**/sdkconfig
+**/sdkconfig.old

code/components/esp32-camera-master/CMakeLists.txt

@@ -1,15 +1,19 @@
-if(IDF_TARGET STREQUAL "esp32")
+if(IDF_TARGET STREQUAL "esp32" OR IDF_TARGET STREQUAL "esp32s2" OR IDF_TARGET STREQUAL "esp32s3")
   set(COMPONENT_SRCS
-    driver/camera.c
+    driver/esp_camera.c
+    driver/cam_hal.c
     driver/sccb.c
     driver/sensor.c
-    driver/xclk.c
     sensors/ov2640.c
     sensors/ov3660.c
     sensors/ov5640.c
     sensors/ov7725.c
     sensors/ov7670.c
     sensors/nt99141.c
+    sensors/gc0308.c
+    sensors/gc2145.c
+    sensors/gc032a.c
+    sensors/bf3005.c
     conversions/yuv.c
     conversions/to_jpg.cpp
     conversions/to_bmp.c
@@ -26,8 +30,34 @@ if(IDF_TARGET STREQUAL "esp32")
     driver/private_include
     sensors/private_include
     conversions/private_include
+    target/private_include
     )
 
+  if(IDF_TARGET STREQUAL "esp32")
+    list(APPEND COMPONENT_SRCS
+      target/xclk.c
+      target/esp32/ll_cam.c
+      )
+  endif()
+
+  if(IDF_TARGET STREQUAL "esp32s2")
+    list(APPEND COMPONENT_SRCS
+      target/xclk.c
+      target/esp32s2/ll_cam.c
+      target/esp32s2/tjpgd.c
+      )
+
+    list(APPEND COMPONENT_PRIV_INCLUDEDIRS
+      target/esp32s2/private_include
+      )
+  endif()
+
+  if(IDF_TARGET STREQUAL "esp32s3")
+    list(APPEND COMPONENT_SRCS
+      target/esp32s3/ll_cam.c
+      )
+  endif()
+
   set(COMPONENT_REQUIRES driver)
   set(COMPONENT_PRIV_REQUIRES freertos nvs_flash)
 

code/components/esp32-camera-master/Kconfig

@@ -5,11 +5,11 @@ menu "Camera configuration"
         default y
         help
             Enable this option if you want to use the OV7670.
-            Disable this option to safe memory.
+            Disable this option to save memory.
 
     config OV7725_SUPPORT
-        bool "Support OV7725 SVGA"
-        default n
+        bool "Support OV7725 VGA"
+        default y
         help
             Enable this option if you want to use the OV7725.
             Disable this option to save memory.
@@ -42,6 +42,34 @@ menu "Camera configuration"
             Enable this option if you want to use the OV5640.
             Disable this option to save memory.
 
+    config GC2145_SUPPORT
+        bool "Support GC2145 2MP"
+        default y
+        help
+            Enable this option if you want to use the GC2145.
+            Disable this option to save memory.
+
+    config GC032A_SUPPORT
+        bool "Support GC032A VGA"
+        default y
+        help
+            Enable this option if you want to use the GC032A.
+            Disable this option to save memory.
+
+    config GC0308_SUPPORT
+        bool "Support GC0308 VGA"
+        default y
+        help
+            Enable this option if you want to use the GC0308.
+            Disable this option to save memory.
+            
+    config BF3005_SUPPORT
+        bool "Support BF3005(BYD3005) VGA"
+        default y
+        help
+            Enable this option if you want to use the BF3005.
+            Disable this option to save memory.
+
     choice SCCB_HARDWARE_I2C_PORT
         bool "I2C peripheral to use for SCCB"
         default SCCB_HARDWARE_I2C_PORT1
@@ -53,6 +81,28 @@ menu "Camera configuration"
 
     endchoice
 
+    config SCCB_CLK_FREQ
+    int "SCCB clk frequency"
+    default 100000
+    range 100000 400000
+    help
+        Increasing this value can reduce the initialization time of the sensor.
+        Please refer to the relevant instructions of the sensor to adjust the value.
+    
+    choice GC_SENSOR_WINDOW_MODE
+        bool "GalaxyCore Sensor Window Mode"
+        depends on (GC2145_SUPPORT || GC032A_SUPPORT || GC0308_SUPPORT)
+        default GC_SENSOR_SUBSAMPLE_MODE
+        help
+            This option determines how to reduce the output size when the resolution you set is less than the maximum resolution.
+            SUBSAMPLE_MODE has a bigger perspective and WINDOWING_MODE has a higher frame rate.
+
+        config GC_SENSOR_WINDOWING_MODE
+            bool "Windowing Mode"
+        config GC_SENSOR_SUBSAMPLE_MODE
+            bool "Subsample Mode"
+    endchoice
+
     choice CAMERA_TASK_PINNED_TO_CORE
         bool "Camera task pinned to core"
         default CAMERA_CORE0
@@ -68,4 +118,12 @@ menu "Camera configuration"
 
     endchoice
 
+    config CAMERA_DMA_BUFFER_SIZE_MAX
+        int "DMA buffer size"
+        range 8192 32768
+        default 32768
+        help
+            Maximum value of DMA buffer
+            Larger values may fail to allocate due to insufficient contiguous memory blocks, and smaller value may cause DMA interrupt to be too frequent
+
 endmenu

code/components/esp32-camera-master/README.md

@@ -1,8 +1,30 @@
 # ESP32 Camera Driver
 
+[![Build examples](https://github.com/espressif/esp32-camera/actions/workflows/build.yml/badge.svg)](https://github.com/espressif/esp32-camera/actions/workflows/build.yml)
 ## General Information
 
-This repository hosts ESP32 compatible driver for OV2640, OV3660, OV5640, OV7670 and OV7725 image sensors. Additionally it provides a few tools, which allow converting the captured frame data to the more common BMP and JPEG formats.
+This repository hosts ESP32 series Soc compatible driver for image sensors. Additionally it provides a few tools, which allow converting the captured frame data to the more common BMP and JPEG formats.
+
+### Supported Soc
+
+- ESP32
+- ESP32-S2
+- ESP32-S3
+
+### Supported Sensor
+
+| model   | max resolution | color type | output format                                                | Len Size |
+| ------- | -------------- | ---------- | ------------------------------------------------------------ | -------- |
+| OV2640  | 1600 x 1200    | color      | YUV(422/420)/YCbCr422<br>RGB565/555<br>8-bit compressed data<br>8/10-bit Raw RGB data | 1/4"     |
+| OV3660  | 2048 x 1536    | color      | raw RGB data<br/>RGB565/555/444<br/>CCIR656<br/>YCbCr422<br/>compression | 1/5"     |
+| OV5640  | 2592 x 1944    | color      | RAW RGB<br/>RGB565/555/444<br/>CCIR656<br/>YUV422/420<br/>YCbCr422<br/>compression | 1/4"     |
+| OV7670  | 640 x 480      | color      | Raw Bayer RGB<br/>Processed Bayer RGB<br>YUV/YCbCr422<br>GRB422<br>RGB565/555 | 1/6"     |
+| OV7725  | 640 x 480      | color      | Raw RGB<br/>GRB 422<br/>RGB565/555/444<br/>YCbCr 422         | 1/4"     |
+| NT99141 | 1280 x 720     | color      | YCbCr 422<br/>RGB565/555/444<br/>Raw<br/>CCIR656<br/>JPEG compression | 1/4"     |
+| GC032A  | 640 x 480      | color      | YUV/YCbCr422<br/>RAW Bayer<br/>RGB565                        | 1/10"    |
+| GC0308  | 640 x 480      | color      | YUV/YCbCr422<br/>RAW Bayer<br/>RGB565                        | 1/6.5"   |
+| GC2145  | 1600 x 1200    | color      | YUV/YCbCr422<br/>RAW Bayer<br/>RGB565                        | 1/5"     |
+| BF3005  | 640 x 480      | color      | YUV/YCbCr422<br/>RAW Bayer<br/>RGB565                        | 1/4"     |
 
 ## Important to Remember
 
@@ -17,7 +39,7 @@ This repository hosts ESP32 compatible driver for OV2640, OV3660, OV5640, OV7670
 ### Using esp-idf
 
 - Clone or download and extract the repository to the components folder of your ESP-IDF project
-- Enable PSRAM in `menuconfig`
+- Enable PSRAM in `menuconfig` (also set Flash and PSRAM frequiencies to 80MHz)
 - Include `esp_camera.h` in your code
 
 ### Using PlatformIO
@@ -75,17 +97,6 @@ However with a bit of patience and experimenting you'll figure the Kconfig out.
 
 If you miss-skip-ignore this critical step the camera module will compile but camera logic inside the library will be 'empty' because the Kconfig sets the proper #ifdef statements during the build process to initialize the selected cameras.  It's very not optional! 
 
-### Kconfig options
-
-| config                            | description                                                                                                                                                  | default                        |
-| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------ |
-| CONFIG_OV2640_SUPPORT             | Support for OV2640 camera                                                                                                                                    | enabled                        |
-| CONFIG_OV7725_SUPPORT             | Support for OV7725 camera                                                                                                                                    | disabled                       |
-| CONFIG_OV3660_SUPPORT             | Support for OV3660 camera                                                                                                                                    | enabled                        |
-| CONFIG_OV5640_SUPPORT             | Support for OV5640 camera                                                                                                                                    | enabled                        |
-| CONFIG_SCCB_HARDWARE_I2C          | Enable this option if you want to use hardware I2C to control the camera. Disable this option to use software I2C.                                           | enabled                        |
-| CONFIG_SCCB_HARDWARE_I2C_PORT     | I2C peripheral to use for SCCB. Can be I2C0 and I2C1.                                                                                                        | CONFIG_SCCB_HARDWARE_I2C_PORT1 |
-| CONFIG_CAMERA_TASK_PINNED_TO_CORE | Pin the camera handle task to a certain core(0/1). It can also be done automatically choosing NO_AFFINITY. Can be CAMERA_CORE0, CAMERA_CORE1 or NO_AFFINITY. | CONFIG_CAMERA_CORE0            |
 
 ## Examples
 
@@ -132,8 +143,7 @@ static camera_config_t camera_config = {
     .pin_href = CAM_PIN_HREF,
     .pin_pclk = CAM_PIN_PCLK,
 
-    //XCLK 20MHz or 10MHz for OV2640 double FPS (Experimental)
-    .xclk_freq_hz = 20000000,
+    .xclk_freq_hz = 20000000,//EXPERIMENTAL: Set to 16MHz on ESP32-S2 or ESP32-S3 to enable EDMA mode
     .ledc_timer = LEDC_TIMER_0,
     .ledc_channel = LEDC_CHANNEL_0,
 
@@ -141,7 +151,8 @@ static camera_config_t camera_config = {
     .frame_size = FRAMESIZE_UXGA,//QQVGA-QXGA Do not use sizes above QVGA when not JPEG
 
     .jpeg_quality = 12, //0-63 lower number means higher quality
-    .fb_count = 1 //if more than one, i2s runs in continuous mode. Use only with JPEG
+    .fb_count = 1, //if more than one, i2s runs in continuous mode. Use only with JPEG
+    .grab_mode = CAMERA_GRAB_WHEN_EMPTY//CAMERA_GRAB_LATEST. Sets when buffers should be filled
 };
 
 esp_err_t camera_init(){

code/components/esp32-camera-master/component.mk

@@ -1,4 +1,4 @@
 COMPONENT_ADD_INCLUDEDIRS := driver/include conversions/include
-COMPONENT_PRIV_INCLUDEDIRS := driver/private_include conversions/private_include sensors/private_include
-COMPONENT_SRCDIRS := driver conversions sensors
+COMPONENT_PRIV_INCLUDEDIRS := driver/private_include conversions/private_include sensors/private_include target/private_include
+COMPONENT_SRCDIRS := driver conversions sensors target target/esp32
 CXXFLAGS += -fno-rtti

code/components/esp32-camera-master/conversions/esp_jpg_decode.c

@@ -17,7 +17,11 @@
 #if ESP_IDF_VERSION_MAJOR >= 4 // IDF 4+
 #if CONFIG_IDF_TARGET_ESP32 // ESP32/PICO-D4
 #include "esp32/rom/tjpgd.h"
-#else 
+#elif CONFIG_IDF_TARGET_ESP32S2
+#include "tjpgd.h"
+#elif CONFIG_IDF_TARGET_ESP32S3
+#include "esp32s3/rom/tjpgd.h"
+#else
 #error Target CONFIG_IDF_TARGET is not supported
 #endif
 #else // ESP32 Before IDF 4.0

code/components/esp32-camera-master/conversions/include/img_converters.h

@@ -22,6 +22,7 @@ extern "C" {
 #include <stdint.h>
 #include <stdbool.h>
 #include "esp_camera.h"
+#include "esp_jpg_decode.h"
 
 typedef size_t (* jpg_out_cb)(void * arg, size_t index, const void* data, size_t len);
 
@@ -62,7 +63,8 @@ bool frame2jpg_cb(camera_fb_t * fb, uint8_t quality, jpg_out_cb cb, void * arg);
  * @param height    Height in pixels of the source image
  * @param format    Format of the source image
  * @param quality   JPEG quality of the resulting image
- * @param out       Pointer to be populated with the address of the resulting buffer
+ * @param out       Pointer to be populated with the address of the resulting buffer.
+ *                  You MUST free the pointer once you are done with it.
  * @param out_len   Pointer to be populated with the length of the output buffer
  *
  * @return true on success
@@ -119,6 +121,8 @@ bool frame2bmp(camera_fb_t * fb, uint8_t ** out, size_t * out_len);
  */
 bool fmt2rgb888(const uint8_t *src_buf, size_t src_len, pixformat_t format, uint8_t * rgb_buf);
 
+bool jpg2rgb565(const uint8_t *src, size_t src_len, uint8_t * out, jpg_scale_t scale);
+
 #ifdef __cplusplus
 }
 #endif

code/components/esp32-camera-master/conversions/to_bmp.c

@@ -24,6 +24,10 @@
 #if ESP_IDF_VERSION_MAJOR >= 4 // IDF 4+
 #if CONFIG_IDF_TARGET_ESP32 // ESP32/PICO-D4
 #include "esp32/spiram.h"
+#elif CONFIG_IDF_TARGET_ESP32S2
+#include "esp32s2/spiram.h"
+#elif CONFIG_IDF_TARGET_ESP32S3
+#include "esp32s3/spiram.h"
 #else 
 #error Target CONFIG_IDF_TARGET is not supported
 #endif
@@ -115,6 +119,54 @@ static bool _rgb_write(void * arg, uint16_t x, uint16_t y, uint16_t w, uint16_t
     return true;
 }
 
+static bool _rgb565_write(void * arg, uint16_t x, uint16_t y, uint16_t w, uint16_t h, uint8_t *data)
+{
+    rgb_jpg_decoder * jpeg = (rgb_jpg_decoder *)arg;
+    if(!data){
+        if(x == 0 && y == 0){
+            //write start
+            jpeg->width = w;
+            jpeg->height = h;
+            //if output is null, this is BMP
+            if(!jpeg->output){
+                jpeg->output = (uint8_t *)_malloc((w*h*3)+jpeg->data_offset);
+                if(!jpeg->output){
+                    return false;
+                }
+            }
+        } else {
+            //write end
+        }
+        return true;
+    }
+
+    size_t jw = jpeg->width*3;
+    size_t jw2 = jpeg->width*2;
+    size_t t = y * jw;
+    size_t t2 = y * jw2;
+    size_t b = t + (h * jw);
+    size_t l = x * 2;
+    uint8_t *out = jpeg->output+jpeg->data_offset;
+    uint8_t *o = out;
+    size_t iy, iy2, ix, ix2;
+
+    w = w * 3;
+
+    for(iy=t, iy2=t2; iy<b; iy+=jw, iy2+=jw2) {
+        o = out+iy2+l;
+        for(ix2=ix=0; ix<w; ix+= 3, ix2 +=2) {
+            uint16_t r = data[ix];
+            uint16_t g = data[ix+1];
+            uint16_t b = data[ix+2];
+            uint16_t c = ((r & 0xF8) << 8) | ((g & 0xFC) << 3) | (b >> 3);
+            o[ix2+1] = c>>8;
+            o[ix2] = c&0xff;
+        }
+        data+=w;
+    }
+    return true;
+}
+
 //input buffer
 static uint32_t _jpg_read(void * arg, size_t index, uint8_t *buf, size_t len)
 {
@@ -140,6 +192,21 @@ static bool jpg2rgb888(const uint8_t *src, size_t src_len, uint8_t * out, jpg_sc
     return true;
 }
 
+bool jpg2rgb565(const uint8_t *src, size_t src_len, uint8_t * out, jpg_scale_t scale)
+{
+    rgb_jpg_decoder jpeg;
+    jpeg.width = 0;
+    jpeg.height = 0;
+    jpeg.input = src;
+    jpeg.output = out;
+    jpeg.data_offset = 0;
+
+    if(esp_jpg_decode(src_len, scale, _jpg_read, _rgb565_write, (void*)&jpeg) != ESP_OK){
+        return false;
+    }
+    return true;
+}
+
 bool jpg2bmp(const uint8_t *src, size_t src_len, uint8_t ** out, size_t * out_len)
 {
 

code/components/esp32-camera-master/conversions/to_jpg.cpp

@@ -25,6 +25,10 @@
 #if ESP_IDF_VERSION_MAJOR >= 4 // IDF 4+
 #if CONFIG_IDF_TARGET_ESP32 // ESP32/PICO-D4
 #include "esp32/spiram.h"
+#elif CONFIG_IDF_TARGET_ESP32S2
+#include "esp32s2/spiram.h"
+#elif CONFIG_IDF_TARGET_ESP32S3
+#include "esp32s3/spiram.h"
 #else 
 #error Target CONFIG_IDF_TARGET is not supported
 #endif
@@ -195,7 +199,7 @@ public:
             return true;
         }
         if ((size_t)len > (max_len - index)) {
-            ESP_LOGW(TAG, "JPG output overflow: %d bytes", len - (max_len - index));
+            //ESP_LOGW(TAG, "JPG output overflow: %d bytes (%d,%d,%d)", len - (max_len - index), len, index, max_len);
             len = max_len - index;
         }
         if (len) {
@@ -215,7 +219,7 @@ bool fmt2jpg(uint8_t *src, size_t src_len, uint16_t width, uint16_t height, pixf
 {
     //todo: allocate proper buffer for holding JPEG data
     //this should be enough for CIF frame size
-    int jpg_buf_len = 64*1024;
+    int jpg_buf_len = 128*1024;
 
 
     uint8_t * jpg_buf = (uint8_t *)_malloc(jpg_buf_len);

code/components/esp32-camera-master/driver/cam_hal.c

@@ -0,0 +1,483 @@
+// Copyright 2010-2020 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+#include <string.h>
+#include "esp_heap_caps.h"
+#include "ll_cam.h"
+#include "cam_hal.h"
+
+static const char *TAG = "cam_hal";
+
+static cam_obj_t *cam_obj = NULL;
+
+static const uint32_t JPEG_SOI_MARKER = 0xFFD8FF;  // written in little-endian for esp32
+static const uint16_t JPEG_EOI_MARKER = 0xD9FF;  // written in little-endian for esp32
+
+static int cam_verify_jpeg_soi(const uint8_t *inbuf, uint32_t length)
+{
+    uint32_t sig = *((uint32_t *)inbuf) & 0xFFFFFF;
+    if(sig != JPEG_SOI_MARKER) {
+        for (uint32_t i = 0; i < length; i++) {
+            sig = *((uint32_t *)(&inbuf[i])) & 0xFFFFFF;
+            if (sig == JPEG_SOI_MARKER) {
+                ESP_LOGW(TAG, "SOI: %d", i);
+                return i;
+            }
+        }
+        ESP_LOGW(TAG, "NO-SOI");
+        return -1;
+    }
+    return 0;
+}
+
+static int cam_verify_jpeg_eoi(const uint8_t *inbuf, uint32_t length)
+{
+    int offset = -1;
+    uint8_t *dptr = (uint8_t *)inbuf + length - 2;
+    while (dptr > inbuf) {
+        uint16_t sig = *((uint16_t *)dptr);
+        if (JPEG_EOI_MARKER == sig) {
+            offset = dptr - inbuf;
+            //ESP_LOGW(TAG, "EOI: %d", length - (offset + 2));
+            return offset;
+        }
+        dptr--;
+    }
+    return -1;
+}
+
+static bool cam_get_next_frame(int * frame_pos)
+{
+    if(!cam_obj->frames[*frame_pos].en){
+        for (int x = 0; x < cam_obj->frame_cnt; x++) {
+            if (cam_obj->frames[x].en) {
+                *frame_pos = x;
+                return true;
+            }
+        }
+    } else {
+        return true;
+    }
+    return false;
+}
+
+static bool cam_start_frame(int * frame_pos)
+{
+    if (cam_get_next_frame(frame_pos)) {
+        if(ll_cam_start(cam_obj, *frame_pos)){
+            // Vsync the frame manually
+            ll_cam_do_vsync(cam_obj);
+            uint64_t us = (uint64_t)esp_timer_get_time();
+            cam_obj->frames[*frame_pos].fb.timestamp.tv_sec = us / 1000000UL;
+            cam_obj->frames[*frame_pos].fb.timestamp.tv_usec = us % 1000000UL;
+            return true;
+        }
+    }
+    return false;
+}
+
+void IRAM_ATTR ll_cam_send_event(cam_obj_t *cam, cam_event_t cam_event, BaseType_t * HPTaskAwoken)
+{
+    if (xQueueSendFromISR(cam->event_queue, (void *)&cam_event, HPTaskAwoken) != pdTRUE) {
+        ll_cam_stop(cam);
+        cam->state = CAM_STATE_IDLE;
+        ESP_EARLY_LOGE(TAG, "EV-%s-OVF", cam_event==CAM_IN_SUC_EOF_EVENT ? "EOF" : "VSYNC");
+    }
+}
+
+//Copy fram from DMA dma_buffer to fram dma_buffer
+static void cam_task(void *arg)
+{
+    int cnt = 0;
+    int frame_pos = 0;
+    cam_obj->state = CAM_STATE_IDLE;
+    cam_event_t cam_event = 0;
+    
+    xQueueReset(cam_obj->event_queue);
+
+    while (1) {
+        xQueueReceive(cam_obj->event_queue, (void *)&cam_event, portMAX_DELAY);
+        DBG_PIN_SET(1);
+        switch (cam_obj->state) {
+
+            case CAM_STATE_IDLE: {
+                if (cam_event == CAM_VSYNC_EVENT) {
+                    //DBG_PIN_SET(1);
+                    if(cam_start_frame(&frame_pos)){
+                        cam_obj->frames[frame_pos].fb.len = 0;
+                        cam_obj->state = CAM_STATE_READ_BUF;
+                    }
+                    cnt = 0;
+                }
+            }
+            break;
+
+            case CAM_STATE_READ_BUF: {
+                camera_fb_t * frame_buffer_event = &cam_obj->frames[frame_pos].fb;
+                size_t pixels_per_dma = (cam_obj->dma_half_buffer_size * cam_obj->fb_bytes_per_pixel) / (cam_obj->dma_bytes_per_item * cam_obj->in_bytes_per_pixel);
+                
+                if (cam_event == CAM_IN_SUC_EOF_EVENT) {
+                    if(!cam_obj->psram_mode){
+                        if (cam_obj->fb_size < (frame_buffer_event->len + pixels_per_dma)) {
+                            ESP_LOGW(TAG, "FB-OVF");
+                            ll_cam_stop(cam_obj);
+                            DBG_PIN_SET(0);
+                            continue;
+                        }
+                        frame_buffer_event->len += ll_cam_memcpy(cam_obj,
+                            &frame_buffer_event->buf[frame_buffer_event->len], 
+                            &cam_obj->dma_buffer[(cnt % cam_obj->dma_half_buffer_cnt) * cam_obj->dma_half_buffer_size], 
+                            cam_obj->dma_half_buffer_size);
+                    }
+                    //Check for JPEG SOI in the first buffer. stop if not found
+                    if (cam_obj->jpeg_mode && cnt == 0 && cam_verify_jpeg_soi(frame_buffer_event->buf, frame_buffer_event->len) != 0) {
+                        ll_cam_stop(cam_obj);
+                        cam_obj->state = CAM_STATE_IDLE;
+                    }
+                    cnt++;
+
+                } else if (cam_event == CAM_VSYNC_EVENT) {
+                    //DBG_PIN_SET(1);
+                    ll_cam_stop(cam_obj);
+
+                    if (cnt || !cam_obj->jpeg_mode || cam_obj->psram_mode) {
+                        if (cam_obj->jpeg_mode) {
+                            if (!cam_obj->psram_mode) {
+                                if (cam_obj->fb_size < (frame_buffer_event->len + pixels_per_dma)) {
+                                    ESP_LOGW(TAG, "FB-OVF");
+                                    cnt--;
+                                } else {
+                                    frame_buffer_event->len += ll_cam_memcpy(cam_obj,
+                                        &frame_buffer_event->buf[frame_buffer_event->len], 
+                                        &cam_obj->dma_buffer[(cnt % cam_obj->dma_half_buffer_cnt) * cam_obj->dma_half_buffer_size], 
+                                        cam_obj->dma_half_buffer_size);
+                                }
+                            }
+                            cnt++;
+                        }
+
+                        cam_obj->frames[frame_pos].en = 0;
+
+                        if (cam_obj->psram_mode) {
+                            if (cam_obj->jpeg_mode) {
+                                frame_buffer_event->len = cnt * cam_obj->dma_half_buffer_size;
+                            } else {
+                                frame_buffer_event->len = cam_obj->recv_size;
+                            }
+                        } else if (!cam_obj->jpeg_mode) {
+                            if (frame_buffer_event->len != cam_obj->fb_size) {
+                                cam_obj->frames[frame_pos].en = 1;
+                                ESP_LOGE(TAG, "FB-SIZE: %u != %u", frame_buffer_event->len, cam_obj->fb_size);
+                            }
+                        }
+                        //send frame
+                        if(!cam_obj->frames[frame_pos].en && xQueueSend(cam_obj->frame_buffer_queue, (void *)&frame_buffer_event, 0) != pdTRUE) {
+                            //pop frame buffer from the queue
+                            camera_fb_t * fb2 = NULL;
+                            if(xQueueReceive(cam_obj->frame_buffer_queue, &fb2, 0) == pdTRUE) {
+                                //push the new frame to the end of the queue
+                                if (xQueueSend(cam_obj->frame_buffer_queue, (void *)&frame_buffer_event, 0) != pdTRUE) {
+                                    cam_obj->frames[frame_pos].en = 1;
+                                    ESP_LOGE(TAG, "FBQ-SND");
+                                }
+                                //free the popped buffer
+                                cam_give(fb2);
+                            } else {
+                                //queue is full and we could not pop a frame from it
+                                cam_obj->frames[frame_pos].en = 1;
+                                ESP_LOGE(TAG, "FBQ-RCV");
+                            }
+                        }
+                    }
+
+                    if(!cam_start_frame(&frame_pos)){
+                        cam_obj->state = CAM_STATE_IDLE;
+                    } else {
+                        cam_obj->frames[frame_pos].fb.len = 0;
+                    }
+                    cnt = 0;
+                }
+            }
+            break;
+        }
+        DBG_PIN_SET(0);
+    }
+}
+
+static lldesc_t * allocate_dma_descriptors(uint32_t count, uint16_t size, uint8_t * buffer)
+{
+    lldesc_t *dma = (lldesc_t *)heap_caps_malloc(count * sizeof(lldesc_t), MALLOC_CAP_DMA);
+    if (dma == NULL) {
+        return dma;
+    }
+
+    for (int x = 0; x < count; x++) {
+        dma[x].size = size;
+        dma[x].length = 0;
+        dma[x].sosf = 0;
+        dma[x].eof = 0;
+        dma[x].owner = 1;
+        dma[x].buf = (buffer + size * x);
+        dma[x].empty = (uint32_t)&dma[(x + 1) % count];
+    }
+    return dma;
+}
+
+static esp_err_t cam_dma_config(const camera_config_t *config)
+{
+    bool ret = ll_cam_dma_sizes(cam_obj);
+    if (0 == ret) {
+        return ESP_FAIL;
+    }
+
+    cam_obj->dma_node_cnt = (cam_obj->dma_buffer_size) / cam_obj->dma_node_buffer_size; // Number of DMA nodes
+    cam_obj->frame_copy_cnt = cam_obj->recv_size / cam_obj->dma_half_buffer_size; // Number of interrupted copies, ping-pong copy
+
+    ESP_LOGI(TAG, "buffer_size: %d, half_buffer_size: %d, node_buffer_size: %d, node_cnt: %d, total_cnt: %d", 
+             cam_obj->dma_buffer_size, cam_obj->dma_half_buffer_size, cam_obj->dma_node_buffer_size, cam_obj->dma_node_cnt, cam_obj->frame_copy_cnt);
+
+    cam_obj->dma_buffer = NULL;
+    cam_obj->dma = NULL;
+
+    cam_obj->frames = (cam_frame_t *)heap_caps_calloc(1, cam_obj->frame_cnt * sizeof(cam_frame_t), MALLOC_CAP_DEFAULT);
+    CAM_CHECK(cam_obj->frames != NULL, "frames malloc failed", ESP_FAIL);
+
+    uint8_t dma_align = 0;
+    size_t fb_size = cam_obj->fb_size;
+    if (cam_obj->psram_mode) {
+        dma_align = ll_cam_get_dma_align(cam_obj);
+        if (cam_obj->fb_size < cam_obj->recv_size) {
+            fb_size = cam_obj->recv_size;
+        }
+    }
+
+    /* Allocate memory for frame buffer */
+    size_t alloc_size = fb_size * sizeof(uint8_t) + dma_align;
+    uint32_t _caps = MALLOC_CAP_8BIT;
+    if (CAMERA_FB_IN_DRAM == config->fb_location) {
+        _caps |= MALLOC_CAP_INTERNAL;
+    } else {
+        _caps |= MALLOC_CAP_SPIRAM;
+    }
+    for (int x = 0; x < cam_obj->frame_cnt; x++) {
+        cam_obj->frames[x].dma = NULL;
+        cam_obj->frames[x].fb_offset = 0;
+        cam_obj->frames[x].en = 0;
+        ESP_LOGI(TAG, "Allocating %d Byte frame buffer in %s", alloc_size, _caps & MALLOC_CAP_SPIRAM ? "PSRAM" : "OnBoard RAM");
+        cam_obj->frames[x].fb.buf = (uint8_t *)heap_caps_malloc(alloc_size, _caps);
+        CAM_CHECK(cam_obj->frames[x].fb.buf != NULL, "frame buffer malloc failed", ESP_FAIL);
+        if (cam_obj->psram_mode) {
+            //align PSRAM buffer. TODO: save the offset so proper address can be freed later
+            cam_obj->frames[x].fb_offset = dma_align - ((uint32_t)cam_obj->frames[x].fb.buf & (dma_align - 1));
+            cam_obj->frames[x].fb.buf += cam_obj->frames[x].fb_offset;
+            ESP_LOGI(TAG, "Frame[%d]: Offset: %u, Addr: 0x%08X", x, cam_obj->frames[x].fb_offset, (uint32_t)cam_obj->frames[x].fb.buf);
+            cam_obj->frames[x].dma = allocate_dma_descriptors(cam_obj->dma_node_cnt, cam_obj->dma_node_buffer_size, cam_obj->frames[x].fb.buf);
+            CAM_CHECK(cam_obj->frames[x].dma != NULL, "frame dma malloc failed", ESP_FAIL);
+        }
+        cam_obj->frames[x].en = 1;
+    }
+
+    if (!cam_obj->psram_mode) {
+        cam_obj->dma_buffer = (uint8_t *)heap_caps_malloc(cam_obj->dma_buffer_size * sizeof(uint8_t), MALLOC_CAP_DMA);
+        if(NULL == cam_obj->dma_buffer) {
+            ESP_LOGE(TAG,"%s(%d): DMA buffer %d Byte malloc failed, the current largest free block:%d Byte", __FUNCTION__, __LINE__, 
+                     cam_obj->dma_buffer_size, heap_caps_get_largest_free_block(MALLOC_CAP_DMA));
+            return ESP_FAIL;
+        }
+
+        cam_obj->dma = allocate_dma_descriptors(cam_obj->dma_node_cnt, cam_obj->dma_node_buffer_size, cam_obj->dma_buffer);
+        CAM_CHECK(cam_obj->dma != NULL, "dma malloc failed", ESP_FAIL);
+    }
+
+    return ESP_OK;
+}
+
+esp_err_t cam_init(const camera_config_t *config)
+{
+    CAM_CHECK(NULL != config, "config pointer is invalid", ESP_ERR_INVALID_ARG);
+
+    esp_err_t ret = ESP_OK;
+    cam_obj = (cam_obj_t *)heap_caps_calloc(1, sizeof(cam_obj_t), MALLOC_CAP_DMA);
+    CAM_CHECK(NULL != cam_obj, "lcd_cam object malloc error", ESP_ERR_NO_MEM);
+
+    cam_obj->swap_data = 0;
+    cam_obj->vsync_pin = config->pin_vsync;
+    cam_obj->vsync_invert = true;
+
+    ll_cam_set_pin(cam_obj, config);
+    ret = ll_cam_config(cam_obj, config);
+    CAM_CHECK_GOTO(ret == ESP_OK, "ll_cam initialize failed", err);
+
+#if CAMERA_DBG_PIN_ENABLE
+    PIN_FUNC_SELECT(GPIO_PIN_MUX_REG[DBG_PIN_NUM], PIN_FUNC_GPIO);
+    gpio_set_direction(DBG_PIN_NUM, GPIO_MODE_OUTPUT);
+    gpio_set_pull_mode(DBG_PIN_NUM, GPIO_FLOATING);
+#endif
+
+    ESP_LOGI(TAG, "cam init ok");
+    return ESP_OK;
+
+err:
+    free(cam_obj);
+    cam_obj = NULL;
+    return ESP_FAIL;
+}
+
+esp_err_t cam_config(const camera_config_t *config, framesize_t frame_size, uint16_t sensor_pid)
+{
+    CAM_CHECK(NULL != config, "config pointer is invalid", ESP_ERR_INVALID_ARG);
+    esp_err_t ret = ESP_OK;
+
+    ret = ll_cam_set_sample_mode(cam_obj, (pixformat_t)config->pixel_format, config->xclk_freq_hz, sensor_pid);
+
+    cam_obj->jpeg_mode = config->pixel_format == PIXFORMAT_JPEG;
+#if CONFIG_IDF_TARGET_ESP32
+    cam_obj->psram_mode = false;
+#else
+    cam_obj->psram_mode = (config->xclk_freq_hz == 16000000);
+#endif
+    cam_obj->frame_cnt = config->fb_count;
+    cam_obj->width = resolution[frame_size].width;
+    cam_obj->height = resolution[frame_size].height;
+
+    if(cam_obj->jpeg_mode){
+        cam_obj->recv_size = cam_obj->width * cam_obj->height / 5;
+        cam_obj->fb_size = cam_obj->recv_size;
+    } else {
+        cam_obj->recv_size = cam_obj->width * cam_obj->height * cam_obj->in_bytes_per_pixel;
+        cam_obj->fb_size = cam_obj->width * cam_obj->height * cam_obj->fb_bytes_per_pixel;
+    }
+    
+    ret = cam_dma_config(config);
+    CAM_CHECK_GOTO(ret == ESP_OK, "cam_dma_config failed", err);
+
+    cam_obj->event_queue = xQueueCreate(cam_obj->dma_half_buffer_cnt - 1, sizeof(cam_event_t));
+    CAM_CHECK_GOTO(cam_obj->event_queue != NULL, "event_queue create failed", err);
+
+    size_t frame_buffer_queue_len = cam_obj->frame_cnt;
+    if (config->grab_mode == CAMERA_GRAB_LATEST && cam_obj->frame_cnt > 1) {
+        frame_buffer_queue_len = cam_obj->frame_cnt - 1;
+    }
+    cam_obj->frame_buffer_queue = xQueueCreate(frame_buffer_queue_len, sizeof(camera_fb_t*));
+    CAM_CHECK_GOTO(cam_obj->frame_buffer_queue != NULL, "frame_buffer_queue create failed", err);
+
+    ret = ll_cam_init_isr(cam_obj);
+    CAM_CHECK_GOTO(ret == ESP_OK, "cam intr alloc failed", err);
+
+    
+#if CONFIG_CAMERA_CORE0
+    xTaskCreatePinnedToCore(cam_task, "cam_task", 2048, NULL, configMAX_PRIORITIES - 2, &cam_obj->task_handle, 0);
+#elif CONFIG_CAMERA_CORE1
+    xTaskCreatePinnedToCore(cam_task, "cam_task", 2048, NULL, configMAX_PRIORITIES - 2, &cam_obj->task_handle, 1);
+#else
+    xTaskCreate(cam_task, "cam_task", 2048, NULL, configMAX_PRIORITIES - 2, &cam_obj->task_handle);
+#endif
+
+    ESP_LOGI(TAG, "cam config ok");
+    return ESP_OK;
+
+err:
+    cam_deinit();
+    return ESP_FAIL;
+}
+
+esp_err_t cam_deinit(void)
+{
+    if (!cam_obj) {
+        return ESP_FAIL;
+    }
+
+    cam_stop();
+    if (cam_obj->task_handle) {
+        vTaskDelete(cam_obj->task_handle);
+    }
+    if (cam_obj->event_queue) {
+        vQueueDelete(cam_obj->event_queue);
+    }
+    if (cam_obj->frame_buffer_queue) {
+        vQueueDelete(cam_obj->frame_buffer_queue);
+    }
+    if (cam_obj->dma) {
+        free(cam_obj->dma);
+    }
+    if (cam_obj->dma_buffer) {
+        free(cam_obj->dma_buffer);
+    }
+    if (cam_obj->frames) {
+        for (int x = 0; x < cam_obj->frame_cnt; x++) {
+            free(cam_obj->frames[x].fb.buf - cam_obj->frames[x].fb_offset);
+            if (cam_obj->frames[x].dma) {
+                free(cam_obj->frames[x].dma);
+            }
+        }
+        free(cam_obj->frames);
+    }
+
+    ll_cam_deinit(cam_obj);
+
+    free(cam_obj);
+    cam_obj = NULL;
+    return ESP_OK;
+}
+
+void cam_stop(void)
+{
+    ll_cam_vsync_intr_enable(cam_obj, false);
+    ll_cam_stop(cam_obj);
+}
+
+void cam_start(void)
+{
+    ll_cam_vsync_intr_enable(cam_obj, true);
+}
+
+camera_fb_t *cam_take(TickType_t timeout)
+{
+    camera_fb_t *dma_buffer = NULL;
+    TickType_t start = xTaskGetTickCount();
+    xQueueReceive(cam_obj->frame_buffer_queue, (void *)&dma_buffer, timeout);
+    if (dma_buffer) {
+        if(cam_obj->jpeg_mode){
+            // find the end marker for JPEG. Data after that can be discarded
+            int offset_e = cam_verify_jpeg_eoi(dma_buffer->buf, dma_buffer->len);
+            if (offset_e >= 0) {
+                // adjust buffer length
+                dma_buffer->len = offset_e + sizeof(JPEG_EOI_MARKER);
+                return dma_buffer;
+            } else {
+                ESP_LOGW(TAG, "NO-EOI");
+                cam_give(dma_buffer);
+                return cam_take(timeout - (xTaskGetTickCount() - start));//recurse!!!!
+            }
+        } else if(cam_obj->psram_mode && cam_obj->in_bytes_per_pixel != cam_obj->fb_bytes_per_pixel){
+            //currently this is used only for YUV to GRAYSCALE
+            dma_buffer->len = ll_cam_memcpy(cam_obj, dma_buffer->buf, dma_buffer->buf, dma_buffer->len);
+        }
+        return dma_buffer;
+    } else {
+        ESP_LOGW(TAG, "Failed to get the frame on time!");
+    }
+    return NULL;
+}
+
+void cam_give(camera_fb_t *dma_buffer)
+{
+    for (int x = 0; x < cam_obj->frame_cnt; x++) {
+        if (&cam_obj->frames[x].fb == dma_buffer) {
+            cam_obj->frames[x].en = 1;
+            break;
+        }
+    }
+}

code/components/esp32-camera-master/driver/esp_camera.c

@@ -0,0 +1,421 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "time.h"
+#include "sys/time.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "driver/gpio.h"
+#include "esp_system.h"
+#include "nvs_flash.h"
+#include "nvs.h"
+#include "sensor.h"
+#include "sccb.h"
+#include "cam_hal.h"
+#include "esp_camera.h"
+#include "xclk.h"
+#if CONFIG_OV2640_SUPPORT
+#include "ov2640.h"
+#endif
+#if CONFIG_OV7725_SUPPORT
+#include "ov7725.h"
+#endif
+#if CONFIG_OV3660_SUPPORT
+#include "ov3660.h"
+#endif
+#if CONFIG_OV5640_SUPPORT
+#include "ov5640.h"
+#endif
+#if CONFIG_NT99141_SUPPORT
+#include "nt99141.h"
+#endif
+#if CONFIG_OV7670_SUPPORT
+#include "ov7670.h"
+#endif
+#if CONFIG_GC2145_SUPPORT
+#include "gc2145.h"
+#endif
+#if CONFIG_GC032A_SUPPORT
+#include "gc032a.h"
+#endif
+#if CONFIG_GC0308_SUPPORT
+#include "gc0308.h"
+#endif
+#if CONFIG_BF3005_SUPPORT
+#include "bf3005.h"
+#endif
+
+#if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG)
+#include "esp32-hal-log.h"
+#define TAG ""
+#else
+#include "esp_log.h"
+static const char *TAG = "camera";
+#endif
+
+typedef struct {
+    sensor_t sensor;
+    camera_fb_t fb;
+} camera_state_t;
+
+static const char *CAMERA_SENSOR_NVS_KEY = "sensor";
+static const char *CAMERA_PIXFORMAT_NVS_KEY = "pixformat";
+static camera_state_t *s_state = NULL;
+
+#if CONFIG_IDF_TARGET_ESP32S3 // LCD_CAM module of ESP32-S3 will generate xclk
+#define CAMERA_ENABLE_OUT_CLOCK(v)
+#define CAMERA_DISABLE_OUT_CLOCK()
+#else
+#define CAMERA_ENABLE_OUT_CLOCK(v) camera_enable_out_clock((v))
+#define CAMERA_DISABLE_OUT_CLOCK() camera_disable_out_clock()
+#endif
+
+typedef struct {
+    int (*detect)(int slv_addr, sensor_id_t *id);
+    int (*init)(sensor_t *sensor);
+} sensor_func_t;
+
+static const sensor_func_t g_sensors[] = {
+#if CONFIG_OV7725_SUPPORT
+    {ov7725_detect, ov7725_init},
+#endif
+#if CONFIG_OV7670_SUPPORT
+    {ov7670_detect, ov7670_init},
+#endif
+#if CONFIG_OV2640_SUPPORT
+    {ov2640_detect, ov2640_init},
+#endif
+#if CONFIG_OV3660_SUPPORT
+    {ov3660_detect, ov3660_init},
+#endif
+#if CONFIG_OV5640_SUPPORT
+    {ov5640_detect, ov5640_init},
+#endif
+#if CONFIG_NT99141_SUPPORT
+    {nt99141_detect, nt99141_init},
+#endif
+#if CONFIG_GC2145_SUPPORT
+    {gc2145_detect, gc2145_init},
+#endif
+#if CONFIG_GC032A_SUPPORT
+    {gc032a_detect, gc032a_init},
+#endif
+#if CONFIG_GC0308_SUPPORT
+    {gc0308_detect, gc0308_init},
+#endif
+#if CONFIG_BF3005_SUPPORT
+    {bf3005_detect, bf3005_init},
+#endif
+};
+
+static esp_err_t camera_probe(const camera_config_t *config, camera_model_t *out_camera_model)
+{
+    *out_camera_model = CAMERA_NONE;
+    if (s_state != NULL) {
+        return ESP_ERR_INVALID_STATE;
+    }
+
+    s_state = (camera_state_t *) calloc(sizeof(camera_state_t), 1);
+    if (!s_state) {
+        return ESP_ERR_NO_MEM;
+    }
+
+    if (config->pin_xclk >= 0) {
+        ESP_LOGD(TAG, "Enabling XCLK output");
+        CAMERA_ENABLE_OUT_CLOCK(config);
+    }
+
+    if (config->pin_sscb_sda != -1) {
+        ESP_LOGD(TAG, "Initializing SSCB");
+        SCCB_Init(config->pin_sscb_sda, config->pin_sscb_scl);
+    }
+
+    if (config->pin_pwdn >= 0) {
+        ESP_LOGD(TAG, "Resetting camera by power down line");
+        gpio_config_t conf = { 0 };
+        conf.pin_bit_mask = 1LL << config->pin_pwdn;
+        conf.mode = GPIO_MODE_OUTPUT;
+        gpio_config(&conf);
+
+        // carefull, logic is inverted compared to reset pin
+        gpio_set_level(config->pin_pwdn, 1);
+        vTaskDelay(10 / portTICK_PERIOD_MS);
+        gpio_set_level(config->pin_pwdn, 0);
+        vTaskDelay(10 / portTICK_PERIOD_MS);
+    }
+
+    if (config->pin_reset >= 0) {
+        ESP_LOGD(TAG, "Resetting camera");
+        gpio_config_t conf = { 0 };
+        conf.pin_bit_mask = 1LL << config->pin_reset;
+        conf.mode = GPIO_MODE_OUTPUT;
+        gpio_config(&conf);
+
+        gpio_set_level(config->pin_reset, 0);
+        vTaskDelay(10 / portTICK_PERIOD_MS);
+        gpio_set_level(config->pin_reset, 1);
+        vTaskDelay(10 / portTICK_PERIOD_MS);
+    }
+
+
+    ESP_LOGD(TAG, "Searching for camera address");
+    vTaskDelay(10 / portTICK_PERIOD_MS);
+
+    uint8_t slv_addr = SCCB_Probe();
+
+    if (slv_addr == 0) {
+        CAMERA_DISABLE_OUT_CLOCK();
+        return ESP_ERR_NOT_FOUND;
+    }
+
+    ESP_LOGI(TAG, "Detected camera at address=0x%02x", slv_addr);
+    s_state->sensor.slv_addr = slv_addr;
+    s_state->sensor.xclk_freq_hz = config->xclk_freq_hz;
+
+    /**
+     * Read sensor ID and then initialize sensor
+     * Attention: Some sensors have the same SCCB address. Therefore, several attempts may be made in the detection process
+     */
+    sensor_id_t *id = &s_state->sensor.id;
+    for (size_t i = 0; i < sizeof(g_sensors) / sizeof(sensor_func_t); i++) {
+        if (g_sensors[i].detect(slv_addr, id)) {
+            camera_sensor_info_t *info = esp_camera_sensor_get_info(id);
+            if (NULL != info) {
+                *out_camera_model = info->model;
+                ESP_LOGI(TAG, "Detected %s camera", info->name);
+                g_sensors[i].init(&s_state->sensor);
+                break;
+            }
+        }
+    }
+
+    if (CAMERA_NONE == *out_camera_model) { //If no supported sensors are detected
+        CAMERA_DISABLE_OUT_CLOCK();
+        ESP_LOGE(TAG, "Detected camera not supported.");
+        return ESP_ERR_NOT_SUPPORTED;
+    }
+
+    ESP_LOGI(TAG, "Camera PID=0x%02x VER=0x%02x MIDL=0x%02x MIDH=0x%02x",
+             id->PID, id->VER, id->MIDH, id->MIDL);
+
+    ESP_LOGD(TAG, "Doing SW reset of sensor");
+    vTaskDelay(10 / portTICK_PERIOD_MS);
+    s_state->sensor.reset(&s_state->sensor);
+
+    return ESP_OK;
+}
+
+esp_err_t esp_camera_init(const camera_config_t *config)
+{
+    esp_err_t err;
+    err = cam_init(config);
+    if (err != ESP_OK) {
+        ESP_LOGE(TAG, "Camera init failed with error 0x%x", err);
+        return err;
+    }
+
+    camera_model_t camera_model = CAMERA_NONE;
+    err = camera_probe(config, &camera_model);
+    if (err != ESP_OK) {
+        ESP_LOGE(TAG, "Camera probe failed with error 0x%x(%s)", err, esp_err_to_name(err));
+        goto fail;
+    }
+
+    framesize_t frame_size = (framesize_t) config->frame_size;
+    pixformat_t pix_format = (pixformat_t) config->pixel_format;
+
+    if (PIXFORMAT_JPEG == pix_format && (!camera_sensor[camera_model].support_jpeg)) {
+        ESP_LOGE(TAG, "JPEG format is not supported on this sensor");
+        err = ESP_ERR_NOT_SUPPORTED;
+        goto fail;
+    }
+
+    if (frame_size > camera_sensor[camera_model].max_size) {
+        ESP_LOGW(TAG, "The frame size exceeds the maximum for this sensor, it will be forced to the maximum possible value");
+        frame_size = camera_sensor[camera_model].max_size;
+    }
+
+    err = cam_config(config, frame_size, s_state->sensor.id.PID);
+    if (err != ESP_OK) {
+        ESP_LOGE(TAG, "Camera config failed with error 0x%x", err);
+        goto fail;
+    }
+
+    s_state->sensor.status.framesize = frame_size;
+    s_state->sensor.pixformat = pix_format;
+    ESP_LOGD(TAG, "Setting frame size to %dx%d", resolution[frame_size].width, resolution[frame_size].height);
+    if (s_state->sensor.set_framesize(&s_state->sensor, frame_size) != 0) {
+        ESP_LOGE(TAG, "Failed to set frame size");
+        err = ESP_ERR_CAMERA_FAILED_TO_SET_FRAME_SIZE;
+        goto fail;
+    }
+    s_state->sensor.set_pixformat(&s_state->sensor, pix_format);
+
+    if (s_state->sensor.id.PID == OV2640_PID) {
+        s_state->sensor.set_gainceiling(&s_state->sensor, GAINCEILING_2X);
+        s_state->sensor.set_bpc(&s_state->sensor, false);
+        s_state->sensor.set_wpc(&s_state->sensor, true);
+        s_state->sensor.set_lenc(&s_state->sensor, true);
+    }
+
+    if (pix_format == PIXFORMAT_JPEG) {
+        s_state->sensor.set_quality(&s_state->sensor, config->jpeg_quality);
+    }
+    s_state->sensor.init_status(&s_state->sensor);
+
+    cam_start();
+
+    return ESP_OK;
+
+fail:
+    esp_camera_deinit();
+    return err;
+}
+
+esp_err_t esp_camera_deinit()
+{
+    esp_err_t ret = cam_deinit();
+    CAMERA_DISABLE_OUT_CLOCK();
+    if (s_state) {
+        SCCB_Deinit();
+
+        free(s_state);
+        s_state = NULL;
+    }
+
+    return ret;
+}
+
+#define FB_GET_TIMEOUT (4000 / portTICK_PERIOD_MS)
+
+camera_fb_t *esp_camera_fb_get()
+{
+    if (s_state == NULL) {
+        return NULL;
+    }
+    camera_fb_t *fb = cam_take(FB_GET_TIMEOUT);
+    //set the frame properties
+    if (fb) {
+        fb->width = resolution[s_state->sensor.status.framesize].width;
+        fb->height = resolution[s_state->sensor.status.framesize].height;
+        fb->format = s_state->sensor.pixformat;
+    }
+    return fb;
+}
+
+void esp_camera_fb_return(camera_fb_t *fb)
+{
+    if (s_state == NULL) {
+        return;
+    }
+    cam_give(fb);
+}
+
+sensor_t *esp_camera_sensor_get()
+{
+    if (s_state == NULL) {
+        return NULL;
+    }
+    return &s_state->sensor;
+}
+
+esp_err_t esp_camera_save_to_nvs(const char *key)
+{
+#if ESP_IDF_VERSION_MAJOR > 3
+    nvs_handle_t handle;
+#else
+    nvs_handle handle;
+#endif
+    esp_err_t ret = nvs_open(key, NVS_READWRITE, &handle);
+
+    if (ret == ESP_OK) {
+        sensor_t *s = esp_camera_sensor_get();
+        if (s != NULL) {
+            ret = nvs_set_blob(handle, CAMERA_SENSOR_NVS_KEY, &s->status, sizeof(camera_status_t));
+            if (ret == ESP_OK) {
+                uint8_t pf = s->pixformat;
+                ret = nvs_set_u8(handle, CAMERA_PIXFORMAT_NVS_KEY, pf);
+            }
+            return ret;
+        } else {
+            return ESP_ERR_CAMERA_NOT_DETECTED;
+        }
+        nvs_close(handle);
+        return ret;
+    } else {
+        return ret;
+    }
+}
+
+esp_err_t esp_camera_load_from_nvs(const char *key)
+{
+#if ESP_IDF_VERSION_MAJOR > 3
+    nvs_handle_t handle;
+#else
+    nvs_handle handle;
+#endif
+    uint8_t pf;
+
+    esp_err_t ret = nvs_open(key, NVS_READWRITE, &handle);
+
+    if (ret == ESP_OK) {
+        sensor_t *s = esp_camera_sensor_get();
+        camera_status_t st;
+        if (s != NULL) {
+            size_t size = sizeof(camera_status_t);
+            ret = nvs_get_blob(handle, CAMERA_SENSOR_NVS_KEY, &st, &size);
+            if (ret == ESP_OK) {
+                s->set_ae_level(s, st.ae_level);
+                s->set_aec2(s, st.aec2);
+                s->set_aec_value(s, st.aec_value);
+                s->set_agc_gain(s, st.agc_gain);
+                s->set_awb_gain(s, st.awb_gain);
+                s->set_bpc(s, st.bpc);
+                s->set_brightness(s, st.brightness);
+                s->set_colorbar(s, st.colorbar);
+                s->set_contrast(s, st.contrast);
+                s->set_dcw(s, st.dcw);
+                s->set_denoise(s, st.denoise);
+                s->set_exposure_ctrl(s, st.aec);
+                s->set_framesize(s, st.framesize);
+                s->set_gain_ctrl(s, st.agc);
+                s->set_gainceiling(s, st.gainceiling);
+                s->set_hmirror(s, st.hmirror);
+                s->set_lenc(s, st.lenc);
+                s->set_quality(s, st.quality);
+                s->set_raw_gma(s, st.raw_gma);
+                s->set_saturation(s, st.saturation);
+                s->set_sharpness(s, st.sharpness);
+                s->set_special_effect(s, st.special_effect);
+                s->set_vflip(s, st.vflip);
+                s->set_wb_mode(s, st.wb_mode);
+                s->set_whitebal(s, st.awb);
+                s->set_wpc(s, st.wpc);
+            }
+            ret = nvs_get_u8(handle, CAMERA_PIXFORMAT_NVS_KEY, &pf);
+            if (ret == ESP_OK) {
+                s->set_pixformat(s, pf);
+            }
+        } else {
+            return ESP_ERR_CAMERA_NOT_DETECTED;
+        }
+        nvs_close(handle);
+        return ret;
+    } else {
+        ESP_LOGW(TAG, "Error (%d) opening nvs key \"%s\"", ret, key);
+        return ret;
+    }
+}

code/components/esp32-camera-master/driver/include/esp_camera.h

@@ -38,7 +38,8 @@
         .pixel_format   = PIXFORMAT_JPEG,
         .frame_size     = FRAMESIZE_SVGA,
         .jpeg_quality   = 10,
-        .fb_count       = 2
+        .fb_count       = 2,
+        .grab_mode      = CAMERA_GRAB_WHEN_EMPTY
     };
 
     esp_err_t camera_example_init(){
@@ -74,6 +75,22 @@
 extern "C" {
 #endif
 
+/**
+ * @brief Configuration structure for camera initialization
+ */
+typedef enum {
+    CAMERA_GRAB_WHEN_EMPTY,         /*!< Fills buffers when they are empty. Less resources but first 'fb_count' frames might be old */
+    CAMERA_GRAB_LATEST              /*!< Except when 1 frame buffer is used, queue will always contain the last 'fb_count' frames */
+} camera_grab_mode_t;
+
+/**
+ * @brief Camera frame buffer location 
+ */
+typedef enum {
+    CAMERA_FB_IN_PSRAM,         /*!< Frame buffer is placed in external PSRAM */
+    CAMERA_FB_IN_DRAM           /*!< Frame buffer is placed in internal DRAM */
+} camera_fb_location_t;
+
 /**
  * @brief Configuration structure for camera initialization
  */
@@ -95,7 +112,7 @@ typedef struct {
     int pin_href;                   /*!< GPIO pin for camera HREF line */
     int pin_pclk;                   /*!< GPIO pin for camera PCLK line */
 
-    int xclk_freq_hz;               /*!< Frequency of XCLK signal, in Hz. Either 20KHz or 10KHz for OV2640 double FPS (Experimental) */
+    int xclk_freq_hz;               /*!< Frequency of XCLK signal, in Hz. EXPERIMENTAL: Set to 16MHz on ESP32-S2 or ESP32-S3 to enable EDMA mode */
 
     ledc_timer_t ledc_timer;        /*!< LEDC timer to be used for generating XCLK  */
     ledc_channel_t ledc_channel;    /*!< LEDC channel to be used for generating XCLK  */
@@ -105,6 +122,8 @@ typedef struct {
 
     int jpeg_quality;               /*!< Quality of JPEG output. 0-63 lower means higher quality  */
     size_t fb_count;                /*!< Number of frame buffers to be allocated. If more than one, then each frame will be acquired (double speed)  */
+    camera_fb_location_t fb_location; /*!< The location where the frame buffer will be allocated */
+    camera_grab_mode_t grab_mode;   /*!< When buffers should be filled */
 } camera_config_t;
 
 /**

code/components/esp32-camera-master/driver/include/sensor.h

@@ -11,13 +11,51 @@
 #include <stdint.h>
 #include <stdbool.h>
 
-#define NT99141_PID     (0x14)
-#define OV9650_PID     (0x96)
-#define OV7725_PID     (0x77)
-#define OV2640_PID     (0x26)
-#define OV3660_PID     (0x36)
-#define OV5640_PID     (0x56)
-#define OV7670_PID     (0x76)
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum {
+    OV9650_PID = 0x96,
+    OV7725_PID = 0x77,
+    OV2640_PID = 0x26,
+    OV3660_PID = 0x3660,
+    OV5640_PID = 0x5640,
+    OV7670_PID = 0x76,
+    NT99141_PID = 0x1410,
+    GC2145_PID = 0x2145,
+    GC032A_PID = 0x232a,
+    GC0308_PID = 0x9b,
+    BF3005_PID = 0x30,
+} camera_pid_t;
+
+typedef enum {
+    CAMERA_OV7725,
+    CAMERA_OV2640,
+    CAMERA_OV3660,
+    CAMERA_OV5640,
+    CAMERA_OV7670,
+    CAMERA_NT99141,
+    CAMERA_GC2145,
+    CAMERA_GC032A,
+    CAMERA_GC0308,
+    CAMERA_BF3005,
+    CAMERA_MODEL_MAX,
+    CAMERA_NONE,
+} camera_model_t;
+
+typedef enum {
+    OV2640_SCCB_ADDR   = 0x30,// 0x60 >> 1
+    OV5640_SCCB_ADDR   = 0x3C,// 0x78 >> 1
+    OV3660_SCCB_ADDR   = 0x3C,// 0x78 >> 1
+    OV7725_SCCB_ADDR   = 0x21,// 0x42 >> 1
+    OV7670_SCCB_ADDR   = 0x21,// 0x42 >> 1
+    NT99141_SCCB_ADDR  = 0x2A,// 0x54 >> 1
+    GC2145_SCCB_ADDR   = 0x3C,// 0x78 >> 1
+    GC032A_SCCB_ADDR   = 0x21,// 0x42 >> 1
+    GC0308_SCCB_ADDR   = 0x21,// 0x42 >> 1
+    BF3005_SCCB_ADDR   = 0x6E,
+} camera_sccb_addr_t;
 
 typedef enum {
     PIXFORMAT_RGB565,    // 2BPP/RGB565
@@ -58,6 +96,15 @@ typedef enum {
     FRAMESIZE_INVALID
 } framesize_t;
 
+typedef struct {
+    const camera_model_t model;
+    const char *name;
+    const camera_sccb_addr_t sccb_addr;
+    const camera_pid_t pid;
+    const framesize_t max_size;
+    const bool support_jpeg;
+} camera_sensor_info_t;
+
 typedef enum {
     ASPECT_RATIO_4X3,
     ASPECT_RATIO_3X2,
@@ -101,11 +148,13 @@ typedef struct {
 
 // Resolution table (in sensor.c)
 extern const resolution_info_t resolution[];
+// camera sensor table (in sensor.c)
+extern const camera_sensor_info_t camera_sensor[];
 
 typedef struct {
     uint8_t MIDH;
     uint8_t MIDL;
-    uint8_t PID;
+    uint16_t PID;
     uint8_t VER;
 } sensor_id_t;
 
@@ -190,4 +239,10 @@ typedef struct _sensor {
     int  (*set_xclk)            (sensor_t *sensor, int timer, int xclk);
 } sensor_t;
 
+camera_sensor_info_t *esp_camera_sensor_get_info(sensor_id_t *id);
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* __SENSOR_H__ */

code/components/esp32-camera-master/driver/private_include/cam_hal.h

@@ -0,0 +1,60 @@
+// Copyright 2010-2020 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "esp_camera.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Uninitialize the lcd_cam module
+ *
+ * @param handle Provide handle pointer to release resources
+ *
+ * @return
+ *     - ESP_OK Success
+ *     - ESP_FAIL Uninitialize fail
+ */
+esp_err_t cam_deinit(void);
+
+/**
+ * @brief Initialize the lcd_cam module
+ *
+ * @param config Configurations - see lcd_cam_config_t struct
+ *
+ * @return
+ *     - ESP_OK Success
+ *     - ESP_ERR_INVALID_ARG Parameter error
+ *     - ESP_ERR_NO_MEM No memory to initialize lcd_cam
+ *     - ESP_FAIL Initialize fail
+ */
+esp_err_t cam_init(const camera_config_t *config);
+
+esp_err_t cam_config(const camera_config_t *config, framesize_t frame_size, uint16_t sensor_pid);
+
+void cam_stop(void);
+
+void cam_start(void);
+
+camera_fb_t *cam_take(TickType_t timeout);
+
+void cam_give(camera_fb_t *dma_buffer);
+
+#ifdef __cplusplus
+}
+#endif

code/components/esp32-camera-master/driver/private_include/camera_common.h

@@ -1,49 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-#include <stddef.h>
-#include <stdbool.h>
-#include "esp_err.h"
-#include "esp_intr_alloc.h"
-#include "freertos/FreeRTOS.h"
-#include "freertos/semphr.h"
-#include "freertos/task.h"
-#include "esp_camera.h"
-#include "sensor.h"
-
-#include "esp_system.h"
-#if ESP_IDF_VERSION_MAJOR >= 4 // IDF 4+
-#if CONFIG_IDF_TARGET_ESP32 // ESP32/PICO-D4
-#include "esp32/rom/lldesc.h"
-#else 
-#error Target CONFIG_IDF_TARGET is not supported
-#endif
-#else // ESP32 Before IDF 4.0
-#include "rom/lldesc.h"
-#endif
-
-typedef union {
-    struct {
-        uint8_t sample2;
-        uint8_t unused2;
-        uint8_t sample1;
-        uint8_t unused1;
-    };
-    uint32_t val;
-} dma_elem_t;
-
-typedef enum {
-    /* camera sends byte sequence: s1, s2, s3, s4, ...
-     * fifo receives: 00 s1 00 s2, 00 s2 00 s3, 00 s3 00 s4, ...
-     */
-    SM_0A0B_0B0C = 0,
-    /* camera sends byte sequence: s1, s2, s3, s4, ...
-     * fifo receives: 00 s1 00 s2, 00 s3 00 s4, ...
-     */
-    SM_0A0B_0C0D = 1,
-    /* camera sends byte sequence: s1, s2, s3, s4, ...
-     * fifo receives: 00 s1 00 00, 00 s2 00 00, 00 s3 00 00, ...
-     */
-    SM_0A00_0B00 = 3,
-} i2s_sampling_mode_t;
-

code/components/esp32-camera-master/driver/private_include/sccb.h

@@ -10,6 +10,7 @@
 #define __SCCB_H__
 #include <stdint.h>
 int SCCB_Init(int pin_sda, int pin_scl);
+int SCCB_Deinit(void);
 uint8_t SCCB_Probe();
 uint8_t SCCB_Read(uint8_t slv_addr, uint8_t reg);
 uint8_t SCCB_Write(uint8_t slv_addr, uint8_t reg, uint8_t data);

code/components/esp32-camera-master/driver/private_include/xclk.h

@@ -1,6 +1,8 @@
 #pragma once
 
-#include "camera_common.h"
+#include "esp_system.h"
+
+esp_err_t xclk_timer_conf(int ledc_timer, int xclk_freq_hz);
 
 esp_err_t camera_enable_out_clock();
 

code/components/esp32-camera-master/driver/sccb.c

@@ -11,6 +11,7 @@
 #include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
 #include "sccb.h"
+#include "sensor.h"
 #include <stdio.h>
 #include "sdkconfig.h"
 #if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG)
@@ -24,24 +25,22 @@ static const char* TAG = "sccb";
 
 #include "driver/i2c.h"
 
-#define SCCB_FREQ               100000           /*!< I2C master frequency*/
-#define WRITE_BIT               I2C_MASTER_WRITE /*!< I2C master write */
-#define READ_BIT                I2C_MASTER_READ  /*!< I2C master read */
-#define ACK_CHECK_EN            0x1              /*!< I2C master will check ack from slave*/
-#define ACK_CHECK_DIS           0x0              /*!< I2C master will not check ack from slave */
-#define ACK_VAL                 0x0              /*!< I2C ack value */
-#define NACK_VAL                0x1              /*!< I2C nack value */
+#define SCCB_FREQ               CONFIG_SCCB_CLK_FREQ  /*!< I2C master frequency*/
+#define WRITE_BIT               I2C_MASTER_WRITE      /*!< I2C master write */
+#define READ_BIT                I2C_MASTER_READ       /*!< I2C master read */
+#define ACK_CHECK_EN            0x1                   /*!< I2C master will check ack from slave*/
+#define ACK_CHECK_DIS           0x0                   /*!< I2C master will not check ack from slave */
+#define ACK_VAL                 0x0                   /*!< I2C ack value */
+#define NACK_VAL                0x1                   /*!< I2C nack value */
 #if CONFIG_SCCB_HARDWARE_I2C_PORT1
 const int SCCB_I2C_PORT         = 1;
 #else
 const int SCCB_I2C_PORT         = 0;
 #endif
-static uint8_t ESP_SLAVE_ADDR   = 0x3c;
 
 int SCCB_Init(int pin_sda, int pin_scl)
 {
-    ESP_LOGI(TAG, "pin_sda %d pin_scl %d\n", pin_sda, pin_scl);
-    //log_i("SCCB_Init start");
+    ESP_LOGI(TAG, "pin_sda %d pin_scl %d", pin_sda, pin_scl);
     i2c_config_t conf;
     memset(&conf, 0, sizeof(i2c_config_t));
     conf.mode = I2C_MODE_MASTER;
@@ -56,10 +55,30 @@ int SCCB_Init(int pin_sda, int pin_scl)
     return 0;
 }
 
-uint8_t SCCB_Probe()
+int SCCB_Deinit(void)
+{
+    return i2c_driver_delete(SCCB_I2C_PORT);
+}
+
+uint8_t SCCB_Probe(void)
 {
     uint8_t slave_addr = 0x0;
-    while(slave_addr < 0x7f) {
+    // for (size_t i = 1; i < 0x80; i++) {
+    //     i2c_cmd_handle_t cmd = i2c_cmd_link_create();
+    //     i2c_master_start(cmd);
+    //     i2c_master_write_byte(cmd, ( i << 1 ) | WRITE_BIT, ACK_CHECK_EN);
+    //     i2c_master_stop(cmd);
+    //     esp_err_t ret = i2c_master_cmd_begin(SCCB_I2C_PORT, cmd, 1000 / portTICK_RATE_MS);
+    //     i2c_cmd_link_delete(cmd);
+    //     if( ret == ESP_OK) {
+    //         ESP_LOGW(TAG, "Found I2C Device at 0x%02X", i);
+    //     }
+    // }
+    for (size_t i = 0; i < CAMERA_MODEL_MAX; i++) {
+        if (slave_addr == camera_sensor[i].sccb_addr) {
+            continue;
+        }
+        slave_addr = camera_sensor[i].sccb_addr;
         i2c_cmd_handle_t cmd = i2c_cmd_link_create();
         i2c_master_start(cmd);
         i2c_master_write_byte(cmd, ( slave_addr << 1 ) | WRITE_BIT, ACK_CHECK_EN);
@@ -67,12 +86,10 @@ uint8_t SCCB_Probe()
         esp_err_t ret = i2c_master_cmd_begin(SCCB_I2C_PORT, cmd, 1000 / portTICK_RATE_MS);
         i2c_cmd_link_delete(cmd);
         if( ret == ESP_OK) {
-            ESP_SLAVE_ADDR = slave_addr;
-            return ESP_SLAVE_ADDR;
+            return slave_addr;
         }
-        slave_addr++;
     }
-    return ESP_SLAVE_ADDR;
+    return 0;
 }
 
 uint8_t SCCB_Read(uint8_t slv_addr, uint8_t reg)

code/components/esp32-camera-master/driver/sensor.c

@@ -1,5 +1,20 @@
+#include <stdio.h>
 #include "sensor.h"
 
+const camera_sensor_info_t camera_sensor[CAMERA_MODEL_MAX] = {
+    // The sequence must be consistent with camera_model_t
+    {CAMERA_OV7725, "OV7725", OV7725_SCCB_ADDR, OV7725_PID, FRAMESIZE_VGA, false},
+    {CAMERA_OV2640, "OV2640", OV2640_SCCB_ADDR, OV2640_PID, FRAMESIZE_UXGA, true},
+    {CAMERA_OV3660, "OV3660", OV3660_SCCB_ADDR, OV3660_PID, FRAMESIZE_QXGA, true},
+    {CAMERA_OV5640, "OV5640", OV5640_SCCB_ADDR, OV5640_PID, FRAMESIZE_QSXGA, true},
+    {CAMERA_OV7670, "OV7670", OV7670_SCCB_ADDR, OV7670_PID, FRAMESIZE_VGA, false},
+    {CAMERA_NT99141, "NT99141", NT99141_SCCB_ADDR, NT99141_PID, FRAMESIZE_HD, true},
+    {CAMERA_GC2145, "GC2145", GC2145_SCCB_ADDR, GC2145_PID, FRAMESIZE_UXGA, false},
+    {CAMERA_GC032A, "GC032A", GC032A_SCCB_ADDR, GC032A_PID, FRAMESIZE_VGA, false},
+    {CAMERA_GC0308, "GC0308", GC0308_SCCB_ADDR, GC0308_PID, FRAMESIZE_VGA, false},
+    {CAMERA_BF3005, "BF3005", BF3005_SCCB_ADDR, BF3005_PID, FRAMESIZE_VGA, false},
+};
+
 const resolution_info_t resolution[FRAMESIZE_INVALID] = {
     {   96,   96, ASPECT_RATIO_1X1   }, /* 96x96 */
     {  160,  120, ASPECT_RATIO_4X3   }, /* QQVGA */
@@ -26,3 +41,13 @@ const resolution_info_t resolution[FRAMESIZE_INVALID] = {
     { 1088, 1920, ASPECT_RATIO_9X16  }, /* Portrait FHD   */
     { 2560, 1920, ASPECT_RATIO_4X3   }, /* QSXGA  */
 };
+
+camera_sensor_info_t *esp_camera_sensor_get_info(sensor_id_t *id)
+{
+    for (int i = 0; i < CAMERA_MODEL_MAX; i++) {
+        if (id->PID == camera_sensor[i].pid) {
+            return (camera_sensor_info_t *)&camera_sensor[i];
+        }
+    }
+    return NULL;
+}

code/components/esp32-camera-master/examples/CMakeLists.txt

@@ -0,0 +1,9 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(EXTRA_COMPONENT_DIRS "../")
+
+add_compile_options(-fdiagnostics-color=always)
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(camera_example)

code/components/esp32-camera-master/examples/main/CMakeLists.txt

@@ -0,0 +1,3 @@
+set(COMPONENT_SRCS take_picture.c)
+set(COMPONENT_ADD_INCLUDEDIRS .)
+register_component()

code/components/esp32-camera-master/examples/main/component.mk

@@ -0,0 +1,5 @@
+#
+# "main" pseudo-component makefile.
+#
+# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
+

code/components/esp32-camera-master/examples/main/take_picture.c

@@ -29,7 +29,6 @@
 
 // ================================ CODE ======================================
 
-#include <esp_event_loop.h>
 #include <esp_log.h>
 #include <esp_system.h>
 #include <nvs_flash.h>
@@ -41,6 +40,8 @@
 
 #include "esp_camera.h"
 
+#define BOARD_WROVER_KIT 1
+
 // WROVER-KIT PIN Map
 #ifdef BOARD_WROVER_KIT
 
@@ -113,11 +114,12 @@ static camera_config_t camera_config = {
     .ledc_timer = LEDC_TIMER_0,
     .ledc_channel = LEDC_CHANNEL_0,
 
-    .pixel_format = PIXFORMAT_JPEG, //YUV422,GRAYSCALE,RGB565,JPEG
-    .frame_size = FRAMESIZE_VGA,    //QQVGA-UXGA Do not use sizes above QVGA when not JPEG
+    .pixel_format = PIXFORMAT_RGB565, //YUV422,GRAYSCALE,RGB565,JPEG
+    .frame_size = FRAMESIZE_QVGA,    //QQVGA-UXGA Do not use sizes above QVGA when not JPEG
 
     .jpeg_quality = 12, //0-63 lower number means higher quality
-    .fb_count = 1       //if more than one, i2s runs in continuous mode. Use only with JPEG
+    .fb_count = 1,       //if more than one, i2s runs in continuous mode. Use only with JPEG
+    .grab_mode = CAMERA_GRAB_WHEN_EMPTY,
 };
 
 static esp_err_t init_camera()
@@ -135,7 +137,9 @@ static esp_err_t init_camera()
 
 void app_main()
 {
-    init_camera();
+    if(ESP_OK != init_camera()) {
+        return;
+    }
 
     while (1)
     {
@@ -144,7 +148,8 @@ void app_main()
 
         // use pic->buf to access the image
         ESP_LOGI(TAG, "Picture taken! Its size was: %zu bytes", pic->len);
+        esp_camera_fb_return(pic);
 
         vTaskDelay(5000 / portTICK_RATE_MS);
     }
-}
+}

code/components/esp32-camera-master/examples/sdkconfig.defaults

@@ -0,0 +1,17 @@
+CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32S2_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y
+
+CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
+CONFIG_PARTITION_TABLE_OFFSET=0x10000
+
+CONFIG_FREERTOS_HZ=1000
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
+
+CONFIG_SPIRAM_SUPPORT=y
+CONFIG_ESP32_SPIRAM_SUPPORT=y
+CONFIG_ESP32S2_SPIRAM_SUPPORT=y
+CONFIG_ESP32S3_SPIRAM_SUPPORT=y
+CONFIG_SPIRAM_SPEED_80M=y
+

code/components/esp32-camera-master/idf_component.yml

@@ -1,5 +1,5 @@
-name: "esp32-camera"
-
-version: "1.0.0"
-
-description: This package hosts ESP32 compatible driver for OV2640 image sensors. Additionally it provides a few tools, which allow converting the captured frame data to the more common BMP and JPEG formats.
+description: ESP32 compatible driver for OV2640, OV3660, OV5640, OV7670 and OV7725 image sensors.
+targets:
+  - esp32
+  - esp32s2
+  - esp32s3

code/components/esp32-camera-master/library.json

@@ -1,6 +1,6 @@
 {
   "name": "esp32-camera",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "keywords": "esp32, camera, espressif, esp32-cam",
   "description": "ESP32 compatible driver for OV2640, OV3660, OV5640, OV7670 and OV7725 image sensors.",
   "repository": {
@@ -16,6 +16,7 @@
       "-Idriver/private_include",
       "-Iconversions/private_include",
       "-Isensors/private_include",
+      "-Itarget/private_include",
       "-fno-rtti"
     ],
     "includeDir": ".",

code/components/esp32-camera-master/sensors/bf3005.c

@@ -0,0 +1,541 @@
+/*
+ * This file is part of the OpenMV project.
+ * Copyright (c) 2013/2014 Ibrahim Abdelkader <i.abdalkader@gmail.com>
+ * This work is licensed under the MIT license, see the file LICENSE for details.
+ *
+ * BF3005 driver.
+ * 
+ * Copyright 2015-2021 Espressif Systems (Shanghai) PTE LTD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include "sccb.h"
+#include "xclk.h"
+#include "bf3005.h"
+#include "bf3005_regs.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+
+#if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG)
+#include "esp32-hal-log.h"
+#else
+#include "esp_log.h"
+static const char* TAG = "bf3005";
+#endif
+
+static const uint8_t default_regs[][2] = {
+  {0x12, 0x40}, //soft reset
+  {0xff, 0xff}, //delay 
+  {0xff, 0xff}, //delay 
+  {0xff, 0xff}, //delay 
+  {0xff, 0xff}, //delay 
+  {0x13, 0x10},	
+  {0x8c, 0x00},
+  {0x8d, 0x64},
+  {0x87, 0x10},
+  {0x13, 0x17},
+  {0x00, 0x20},
+  {0x01, 0x1a},
+  {0x02, 0x22},
+  {0x09, 0x03},
+  {0x0c, 0x80},
+  {0x0d, 0x24},
+  {0x0e, 0x21},
+  {0x0f, 0x28},
+  {0x11, 0x08},
+  {0x15, 0x10}, // 0X10
+  {0x16, 0x03},
+  {0x1e, 0x30},
+  {0x20, 0x8a},
+  {0x21, 0x03},
+  {0x23, 0x55},
+  {0x24, 0x68},
+  {0x25, 0x78},
+  {0x2a, 0x00},
+  {0x2b, 0x00},
+  {0x2d, 0x4f},
+  {0x2e, 0x98},
+  {0x2f, 0x04},
+  {0x30, 0xad},
+  {0x31, 0x17},
+  {0x32, 0x6e},
+  {0x33, 0x20},
+  {0x35, 0xa6},
+  {0x3b, 0x00},
+  {0x3e, 0x00},
+  {0x3f, 0xA8},
+  {0x40, 0x38},
+  {0x41, 0x32},
+  {0x42, 0x2b},
+  {0x43, 0x26},
+  {0x44, 0x1a},
+  {0x45, 0x16},
+  {0x46, 0x10},
+  {0x47, 0x0f},
+  {0x48, 0x0c},
+  {0x49, 0x0a},
+  {0x4b, 0x09},
+  {0x4c, 0x08},
+  {0x4d, 0x3c},
+  {0x4e, 0x06},
+  {0x4f, 0x05},
+  {0x50, 0x03},
+  {0x51, 0x25},
+  {0x52, 0x88},
+  {0x53, 0x03},
+  {0x63, 0x20},
+  {0x64, 0x02},
+  {0x65, 0xa6},
+  {0x66, 0xb6},
+  {0x69, 0x00},
+  {0x70, 0xFF},
+  {0x71, 0xa6},
+  {0x72, 0x2f},
+  {0x73, 0x2f},
+  {0x74, 0x2F},
+  {0x75, 0x0e},
+  {0x76, 0x1e},
+  {0x77, 0x00},
+  {0x78, 0x1e},
+  {0x79, 0x8a},
+  {0x7d, 0xe2},
+  {0x80, 0x44},
+  {0x81, 0x00},
+  {0x82, 0x18},
+  {0x83, 0x1b},
+  {0x84, 0x24},
+  {0x85, 0x2a},
+  {0x86, 0x4f},
+  {0x89, 0x82}, //0x82
+  {0x8b, 0x02},
+  {0x8e, 0x03},
+  {0x8f, 0xFC},
+  {0x9d, 0x4d},
+  {0x9e, 0x41},
+  {0xa1, 0x21},
+  {0xa2, 0x12},
+  {0xa3, 0x32},
+  {0xa4, 0x05},
+  {0xa5, 0x32},
+  {0xa6, 0x04},
+  {0xa7, 0x7f},
+  {0xa8, 0x7f},
+  {0xa9, 0x21},
+  {0xaa, 0x21},
+  {0xab, 0x21},
+  {0xac, 0x0a},
+  {0xad, 0xf0},
+  {0xae, 0xff},
+  {0xaf, 0x1d},
+  {0xb0, 0x94},
+  {0xb1, 0xc0},
+  {0xb2, 0xc0},
+  {0xd2, 0x30},
+  {0xe0, 0x0d},
+  {0xe1, 0x44},
+  {0xe7, 0x7c},
+  {0xe8, 0x89},
+  {0xe9, 0x01},
+  {0xea, 0x01},
+  {0xf0, 0x01},
+  {0xf3, 0x49},
+  {0xf4, 0xff},
+  {0xf5, 0x01},
+  {0xf6, 0xf2},
+  {0xf7, 0x6f},
+  {0x1b, 0x80},
+  {0x00, 0x00},
+};
+
+static int get_reg(sensor_t *sensor, int reg, int mask)
+{
+    int ret = SCCB_Read(sensor->slv_addr, reg & 0xFF);
+    if(ret > 0){
+        ret &= mask;
+    }
+    return ret;
+}
+
+static int set_reg(sensor_t *sensor, int reg, int mask, int value)
+{
+    int ret = 0;
+    ret = SCCB_Read(sensor->slv_addr, reg & 0xFF);
+    if(ret < 0){
+        return ret;
+    }
+    value = (ret & ~mask) | (value & mask);
+    ret = SCCB_Write(sensor->slv_addr, reg & 0xFF, value);
+    return ret;
+}
+
+static int set_reg_bits(sensor_t *sensor, uint8_t reg, uint8_t offset, uint8_t length, uint8_t value)
+{
+    int ret = 0;
+    ret = SCCB_Read(sensor->slv_addr, reg);
+    if(ret < 0){
+        return ret;
+    }
+    uint8_t mask = ((1 << length) - 1) << offset;
+    value = (ret & ~mask) | ((value << offset) & mask);
+    ret = SCCB_Write(sensor->slv_addr, reg & 0xFF, value);
+    return ret;
+}
+
+static int get_reg_bits(sensor_t *sensor, uint8_t reg, uint8_t offset, uint8_t length)
+{
+    int ret = 0;
+    ret = SCCB_Read(sensor->slv_addr, reg);
+    if(ret < 0){
+        return ret;
+    }
+    uint8_t mask = ((1 << length) - 1) << offset;
+    return (ret & mask) >> offset;
+}
+
+
+static int reset(sensor_t *sensor)
+{
+    int i=0;
+    const uint8_t (*regs)[2];
+
+    // Write default regsiters
+    for (i=0, regs = default_regs; regs[i][0]; i++) {
+        SCCB_Write(sensor->slv_addr, regs[i][0], regs[i][1]);
+    }
+
+    // Delay
+    vTaskDelay(50 / portTICK_PERIOD_MS);
+
+    return 0;
+}
+
+static int set_pixformat(sensor_t *sensor, pixformat_t pixformat)
+{
+    int ret=0;
+    sensor->pixformat = pixformat;
+
+    switch (pixformat) {
+    case PIXFORMAT_RGB565:
+        set_reg_bits(sensor, 0x12, 2, 1, 1);
+        break;
+    case PIXFORMAT_RAW:
+        set_reg_bits(sensor, 0x12, 0, 3, 0x4);
+        break;
+    case PIXFORMAT_YUV422:
+    case PIXFORMAT_GRAYSCALE:
+       set_reg_bits(sensor, 0x12, 2, 1, 0);
+        break;
+    default:
+        return -1;
+    }
+
+    // Delay
+    vTaskDelay(30 / portTICK_PERIOD_MS);
+
+    return ret;
+}
+
+static int set_framesize(sensor_t *sensor, framesize_t framesize)
+{
+    int ret=0;
+    if (framesize > FRAMESIZE_VGA) {
+        return -1;
+    }
+    uint16_t w = resolution[framesize].width;
+    uint16_t h = resolution[framesize].height;
+    // uint8_t reg = SCCB_Read(sensor->slv_addr, COM7);
+
+    sensor->status.framesize = framesize;
+
+    // Write MSBs
+    ret |= SCCB_Write(sensor->slv_addr, 0x17, 0);
+    ret |= SCCB_Write(sensor->slv_addr, 0x18, w>>2);
+
+    ret |= SCCB_Write(sensor->slv_addr, 0x19, 0);
+    ret |= SCCB_Write(sensor->slv_addr, 0x1a, h>>2);
+
+    // Write LSBs
+    ret |= SCCB_Write(sensor->slv_addr, 0x03, 0);
+    printf("%s %d\r\n", __func__, __LINE__);
+    if((w<=320)&&(h<=240))
+    {
+        printf("%s %d\r\n", __func__, __LINE__);
+        // Enable auto-scaling/zooming factors
+        //ret |= SCCB_Write(sensor->slv_addr, 0x12, 0x50);
+        set_reg_bits(sensor, 0x12, 4, 1, 1);
+
+        ret |= SCCB_Write(sensor->slv_addr, 0x17, (80-w/4));
+        ret |= SCCB_Write(sensor->slv_addr, 0x18, (80+w/4));
+
+        ret |= SCCB_Write(sensor->slv_addr, 0x19, (60-h/4));
+
+        ret |= SCCB_Write(sensor->slv_addr, 0x1a, (60+h/4));
+        ret |= SCCB_Write(sensor->slv_addr, 0x03, 0);
+
+    } else if((w<=640)&&(h<=480))
+    	{
+       // Enable auto-scaling/zooming factors
+        //ret |= SCCB_Write(sensor->slv_addr, 0x12, 0x40);
+        set_reg_bits(sensor, 0x12, 4, 1, 0);
+
+        ret |= SCCB_Write(sensor->slv_addr, 0x17, (80-w/8));
+        ret |= SCCB_Write(sensor->slv_addr, 0x18, (80+w/8));
+
+        ret |= SCCB_Write(sensor->slv_addr, 0x19, (60-h/8));
+
+        ret |= SCCB_Write(sensor->slv_addr, 0x1a, (60+h/8));
+        ret |= SCCB_Write(sensor->slv_addr, 0x03, 0);
+    }
+
+    // Delay
+    vTaskDelay(30 / portTICK_PERIOD_MS);
+
+    return ret;
+}
+
+static int set_colorbar(sensor_t *sensor, int value)
+{
+    int ret=0;
+    sensor->status.colorbar = value;
+
+    ret |= SCCB_Write(sensor->slv_addr, 0xb9, value);
+
+    return ret;
+}
+
+static int set_whitebal(sensor_t *sensor, int enable)
+{
+    if(set_reg_bits(sensor, 0x13, 1, 1, enable) >= 0){
+        sensor->status.awb = !!enable;
+    }
+    return sensor->status.awb;
+}
+
+
+static int set_gain_ctrl(sensor_t *sensor, int enable)
+{
+    if(set_reg_bits(sensor, 0x13, 2, 1, enable) >= 0){
+        sensor->status.agc = !!enable;
+    }
+    return sensor->status.agc;
+}
+
+
+static int set_exposure_ctrl(sensor_t *sensor, int enable)
+{
+    if(set_reg_bits(sensor, 0x13, 0, 1, enable) >= 0){
+        sensor->status.aec = !!enable;
+    }
+    return sensor->status.aec;
+}
+
+static int set_hmirror(sensor_t *sensor, int enable)
+{
+    if(set_reg_bits(sensor, 0x1e, 5, 1, enable) >= 0){
+        sensor->status.hmirror = !!enable;
+    }
+    return sensor->status.hmirror;
+}
+
+static int set_vflip(sensor_t *sensor, int enable)
+{
+    if(set_reg_bits(sensor, 0x1e, 4, 1, enable) >= 0){
+        sensor->status.vflip = !!enable;
+    }
+    return sensor->status.vflip;
+}
+
+static int set_raw_gma_dsp(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    ret = set_reg_bits(sensor, 0xf1, 1, 1, !enable);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set raw_gma to: %d", !enable);
+        sensor->status.raw_gma = !enable;
+    }
+    return ret;
+}
+
+
+static int set_lenc_dsp(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    ret = set_reg_bits(sensor, 0xf1, 0, 1, !enable);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set lenc to: %d", !enable);
+        sensor->status.lenc = !enable;
+    }
+    return ret;
+}
+
+static int set_agc_gain(sensor_t *sensor, int option)
+{
+    int ret = 0;
+    ret = set_reg_bits(sensor, 0x13, 4, 1, !!option);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set gain to: %d", !!option);
+        sensor->status.agc_gain = !!option;
+    }
+    return ret;
+}
+
+static int set_awb_gain_dsp(sensor_t *sensor, int value)
+{
+    int ret = 0;
+    ret = SCCB_Write(sensor->slv_addr, 0xa6, value);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set awb gain threthold to: %d", value);
+        sensor->status.awb_gain = value;
+    }
+    return ret;
+}
+
+static int set_brightness(sensor_t *sensor, int level)
+{
+    int ret = 0;
+    ret = SCCB_Write(sensor->slv_addr, 0x55, level);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set brightness to: %d", level);
+        sensor->status.brightness = level;
+    }
+    return ret;
+}
+
+static int set_contrast(sensor_t *sensor, int level)
+{
+    int ret = 0;
+    ret = SCCB_Write(sensor->slv_addr, 0x56, level);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set contrast to: %d", level);
+        sensor->status.contrast = level;
+    }
+    return ret;
+}
+
+static int set_sharpness(sensor_t *sensor, int level)
+{
+    int ret = 0;
+    ret = SCCB_Write(sensor->slv_addr, 0x70, level);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set sharpness to: %d", level);
+        sensor->status.sharpness = level;
+    }
+    return ret;
+}
+
+static int init_status(sensor_t *sensor)
+{
+    sensor->status.brightness = SCCB_Read(sensor->slv_addr, 0x55);
+    sensor->status.contrast = SCCB_Read(sensor->slv_addr, 0x56);
+    sensor->status.saturation = 0;
+    sensor->status.ae_level = 0;
+    
+    sensor->status.gainceiling = SCCB_Read(sensor->slv_addr, 0x87);
+    sensor->status.awb = get_reg_bits(sensor, 0x13, 1, 1);
+    sensor->status.awb_gain = SCCB_Read(sensor->slv_addr, 0xa6);
+    sensor->status.aec = get_reg_bits(sensor, 0x13, 0, 1);
+
+    sensor->status.agc = get_reg_bits(sensor, 0x13, 2, 1);
+    
+    sensor->status.raw_gma = get_reg_bits(sensor, 0xf1, 1, 1);
+    sensor->status.lenc = get_reg_bits(sensor, 0xf1, 0, 1);
+    sensor->status.hmirror = get_reg_bits(sensor, 0x1e, 5, 1);
+    sensor->status.vflip = get_reg_bits(sensor, 0x1e, 4, 1);
+    
+    sensor->status.colorbar = SCCB_Read(sensor->slv_addr, 0xb9);
+    sensor->status.sharpness = SCCB_Read(sensor->slv_addr, 0x70);
+    
+    return 0;
+}
+
+static int set_dummy(sensor_t *sensor, int val){ return -1; }
+static int set_gainceiling_dummy(sensor_t *sensor, gainceiling_t val){ return -1; }
+static int set_res_raw(sensor_t *sensor, int startX, int startY, int endX, int endY, int offsetX, int offsetY, int totalX, int totalY, int outputX, int outputY, bool scale, bool binning){return -1;}
+static int _set_pll(sensor_t *sensor, int bypass, int multiplier, int sys_div, int root_2x, int pre_div, int seld5, int pclk_manual, int pclk_div){return -1;}
+
+static int set_xclk(sensor_t *sensor, int timer, int xclk)
+{
+    int ret = 0;
+    sensor->xclk_freq_hz = xclk * 1000000U;
+    ret = xclk_timer_conf(timer, sensor->xclk_freq_hz);
+    return ret;
+}
+
+int bf3005_detect(int slv_addr, sensor_id_t *id)
+{
+    if (BF3005_SCCB_ADDR == slv_addr) {
+        uint16_t PID = SCCB_Read(slv_addr, 0xFC);
+        if (BF3005_PID == PID) {
+            id->PID = PID;
+            id->VER = SCCB_Read(slv_addr, 0xFD);
+            id->MIDL = SCCB_Read(slv_addr, 0xFC);
+            id->MIDH = SCCB_Read(slv_addr, 0xFD);
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
+int bf3005_init(sensor_t *sensor)
+{
+    // Set function pointers
+    sensor->reset = reset;
+    sensor->init_status = init_status;
+    sensor->set_pixformat = set_pixformat;
+    sensor->set_framesize = set_framesize;
+    sensor->set_brightness = set_brightness;
+    sensor->set_contrast = set_contrast;
+
+    sensor->set_colorbar = set_colorbar;
+
+    sensor->set_gain_ctrl = set_gain_ctrl;
+    sensor->set_exposure_ctrl = set_exposure_ctrl;
+    sensor->set_hmirror = set_hmirror;
+    sensor->set_vflip = set_vflip;
+
+    sensor->set_whitebal = set_whitebal;
+
+    sensor->set_awb_gain = set_awb_gain_dsp;
+    sensor->set_agc_gain = set_agc_gain;
+    
+    sensor->set_raw_gma = set_raw_gma_dsp;
+    sensor->set_lenc = set_lenc_dsp;
+
+    sensor->set_sharpness = set_sharpness;
+    //not supported
+    sensor->set_saturation= set_dummy;
+    sensor->set_denoise = set_dummy;
+    sensor->set_quality = set_dummy;
+    sensor->set_special_effect = set_dummy;
+    sensor->set_wb_mode = set_dummy;
+    sensor->set_ae_level = set_dummy;
+    sensor->set_gainceiling = set_gainceiling_dummy;
+
+
+    sensor->get_reg = get_reg;
+    sensor->set_reg = set_reg;
+    sensor->set_res_raw = set_res_raw;
+    sensor->set_pll = _set_pll;
+    sensor->set_xclk = set_xclk;
+    
+    ESP_LOGD(TAG, "BF3005 Attached");
+
+    return 0;
+}

code/components/esp32-camera-master/sensors/gc0308.c

@@ -0,0 +1,467 @@
+// Copyright 2015-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "sccb.h"
+#include "gc0308.h"
+#include "gc0308_regs.h"
+#include "gc0308_settings.h"
+
+#if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG)
+#include "esp32-hal-log.h"
+#else
+#include "esp_log.h"
+static const char *TAG = "gc0308";
+#endif
+
+#define H8(v) ((v)>>8)
+#define L8(v) ((v)&0xff)
+
+//#define REG_DEBUG_ON
+
+static int read_reg(uint8_t slv_addr, const uint16_t reg)
+{
+    int ret = SCCB_Read(slv_addr, reg);
+#ifdef REG_DEBUG_ON
+    if (ret < 0) {
+        ESP_LOGE(TAG, "READ REG 0x%04x FAILED: %d", reg, ret);
+    }
+#endif
+    return ret;
+}
+
+static int write_reg(uint8_t slv_addr, const uint16_t reg, uint8_t value)
+{
+    int ret = 0;
+#ifndef REG_DEBUG_ON
+    ret = SCCB_Write(slv_addr, reg, value);
+#else
+    int old_value = read_reg(slv_addr, reg);
+    if (old_value < 0) {
+        return old_value;
+    }
+    if ((uint8_t)old_value != value) {
+        ESP_LOGI(TAG, "NEW REG 0x%04x: 0x%02x to 0x%02x", reg, (uint8_t)old_value, value);
+        ret = SCCB_Write(slv_addr, reg, value);
+    } else {
+        ESP_LOGD(TAG, "OLD REG 0x%04x: 0x%02x", reg, (uint8_t)old_value);
+        ret = SCCB_Write(slv_addr, reg, value);//maybe not?
+    }
+    if (ret < 0) {
+        ESP_LOGE(TAG, "WRITE REG 0x%04x FAILED: %d", reg, ret);
+    }
+#endif
+    return ret;
+}
+
+static int check_reg_mask(uint8_t slv_addr, uint16_t reg, uint8_t mask)
+{
+    return (read_reg(slv_addr, reg) & mask) == mask;
+}
+
+static int set_reg_bits(uint8_t slv_addr, uint16_t reg, uint8_t offset, uint8_t mask, uint8_t value)
+{
+    int ret = 0;
+    uint8_t c_value, new_value;
+    ret = read_reg(slv_addr, reg);
+    if (ret < 0) {
+        return ret;
+    }
+    c_value = ret;
+    new_value = (c_value & ~(mask << offset)) | ((value & mask) << offset);
+    ret = write_reg(slv_addr, reg, new_value);
+    return ret;
+}
+
+static int write_regs(uint8_t slv_addr, const uint16_t (*regs)[2])
+{
+    int i = 0, ret = 0;
+    while (!ret && regs[i][0] != REGLIST_TAIL) {
+        if (regs[i][0] == REG_DLY) {
+            vTaskDelay(regs[i][1] / portTICK_PERIOD_MS);
+        } else {
+            ret = write_reg(slv_addr, regs[i][0], regs[i][1]);
+        }
+        i++;
+    }
+    return ret;
+}
+
+static void print_regs(uint8_t slv_addr)
+{
+#ifdef DEBUG_PRINT_REG
+    ESP_LOGI(TAG, "REG list look ======================");
+    for (size_t i = 0xf0; i <= 0xfe; i++) {
+        ESP_LOGI(TAG, "reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+    ESP_LOGI(TAG, "\npage 0 ===");
+    write_reg(slv_addr, 0xfe, 0x00); // page 0
+    for (size_t i = 0x03; i <= 0xa2; i++) {
+        ESP_LOGI(TAG, "p0 reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+
+    ESP_LOGI(TAG, "\npage 3 ===");
+    write_reg(slv_addr, 0xfe, 0x03); // page 3
+    for (size_t i = 0x01; i <= 0x43; i++) {
+        ESP_LOGI(TAG, "p3 reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+#endif
+}
+
+static int reset(sensor_t *sensor)
+{
+    int ret = 0;
+    // Software Reset: clear all registers and reset them to their default values
+    ret = write_reg(sensor->slv_addr, RESET_RELATED, 0xf0);
+    if (ret) {
+        ESP_LOGE(TAG, "Software Reset FAILED!");
+        return ret;
+    }
+    vTaskDelay(100 / portTICK_PERIOD_MS);
+    ret = write_regs(sensor->slv_addr, gc0308_sensor_default_regs);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Camera defaults loaded");
+        vTaskDelay(100 / portTICK_PERIOD_MS);
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+#ifdef CONFIG_IDF_TARGET_ESP32
+        set_reg_bits(sensor->slv_addr, 0x28, 4, 0x07, 1);  //frequency division for esp32, ensure pclk <= 15MHz
+#endif
+    }
+    return ret;
+}
+
+static int set_pixformat(sensor_t *sensor, pixformat_t pixformat)
+{
+    int ret = 0;
+
+    switch (pixformat) {
+    case PIXFORMAT_RGB565:
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        ret = set_reg_bits(sensor->slv_addr, 0x24, 0, 0x0f, 6);  //RGB565
+        break;
+
+    case PIXFORMAT_YUV422:
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        ret = set_reg_bits(sensor->slv_addr, 0x24, 0, 0x0f, 2); //yuv422 Y Cb Y Cr
+        break;
+    default:
+        ESP_LOGW(TAG, "unsupport format");
+        ret = -1;
+        break;
+    }
+
+    if (ret == 0) {
+        sensor->pixformat = pixformat;
+        ESP_LOGD(TAG, "Set pixformat to: %u", pixformat);
+    }
+    return ret;
+}
+
+static int set_framesize(sensor_t *sensor, framesize_t framesize)
+{
+    int ret = 0;
+    if (framesize > FRAMESIZE_VGA) {
+        ESP_LOGW(TAG, "Invalid framesize: %u", framesize);
+        framesize = FRAMESIZE_VGA;
+    }
+    sensor->status.framesize = framesize;
+    uint16_t w = resolution[framesize].width;
+    uint16_t h = resolution[framesize].height;
+    uint16_t row_s = (resolution[FRAMESIZE_VGA].height - h) / 2;
+    uint16_t col_s = (resolution[FRAMESIZE_VGA].width - w) / 2;
+    (void)row_s;
+    (void)col_s;
+
+#if CONFIG_GC_SENSOR_SUBSAMPLE_MODE
+    struct subsample_cfg {
+        uint16_t ratio_numerator;
+        uint16_t ratio_denominator;
+        uint8_t reg0x54;
+        uint8_t reg0x56;
+        uint8_t reg0x57;
+        uint8_t reg0x58;
+        uint8_t reg0x59;
+    };
+    const struct subsample_cfg subsample_cfgs[] = { // define some subsample ratio
+        {84, 420, 0x55, 0x00, 0x00, 0x00, 0x00}, //1/5
+        {105, 420, 0x44, 0x00, 0x00, 0x00, 0x00},//1/4
+        {140, 420, 0x33, 0x00, 0x00, 0x00, 0x00},//1/3
+        {210, 420, 0x22, 0x00, 0x00, 0x00, 0x00},//1/2
+        {240, 420, 0x77, 0x02, 0x46, 0x02, 0x46},//4/7
+        {252, 420, 0x55, 0x02, 0x04, 0x02, 0x04},//3/5
+        {280, 420, 0x33, 0x02, 0x00, 0x02, 0x00},//2/3
+        {420, 420, 0x11, 0x00, 0x00, 0x00, 0x00},//1/1
+    };
+    uint16_t win_w = 640;
+    uint16_t win_h = 480;
+    const struct subsample_cfg *cfg = NULL;
+    /**
+     * Strategy: try to keep the maximum perspective
+     */
+    for (size_t i = 0; i < sizeof(subsample_cfgs) / sizeof(struct subsample_cfg); i++) {
+        cfg = &subsample_cfgs[i];
+        if ((win_w * cfg->ratio_numerator / cfg->ratio_denominator >= w) && (win_h * cfg->ratio_numerator / cfg->ratio_denominator >= h)) {
+            win_w = w * cfg->ratio_denominator / cfg->ratio_numerator;
+            win_h = h * cfg->ratio_denominator / cfg->ratio_numerator;
+            row_s = (resolution[FRAMESIZE_VGA].height - win_h) / 2;
+            col_s = (resolution[FRAMESIZE_VGA].width - win_w) / 2;
+            ESP_LOGI(TAG, "subsample win:%dx%d, ratio:%f", win_w, win_h, (float)cfg->ratio_numerator / (float)cfg->ratio_denominator);
+            break;
+        }
+    }
+
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+
+    write_reg(sensor->slv_addr, 0x05, H8(row_s));
+    write_reg(sensor->slv_addr, 0x06, L8(row_s));
+    write_reg(sensor->slv_addr, 0x07, H8(col_s));
+    write_reg(sensor->slv_addr, 0x08, L8(col_s));
+    write_reg(sensor->slv_addr, 0x09, H8(win_h + 8));
+    write_reg(sensor->slv_addr, 0x0a, L8(win_h + 8));
+    write_reg(sensor->slv_addr, 0x0b, H8(win_w + 8));
+    write_reg(sensor->slv_addr, 0x0c, L8(win_w + 8));
+
+    write_reg(sensor->slv_addr, 0xfe, 0x01);
+    set_reg_bits(sensor->slv_addr, 0x53, 7, 0x01, 1);
+    set_reg_bits(sensor->slv_addr, 0x55, 0, 0x01, 1);
+    write_reg(sensor->slv_addr, 0x54, cfg->reg0x54);
+    write_reg(sensor->slv_addr, 0x56, cfg->reg0x56);
+    write_reg(sensor->slv_addr, 0x57, cfg->reg0x57);
+    write_reg(sensor->slv_addr, 0x58, cfg->reg0x58);
+    write_reg(sensor->slv_addr, 0x59, cfg->reg0x59);
+
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+
+#elif CONFIG_GC_SENSOR_WINDOWING_MODE
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+
+    write_reg(sensor->slv_addr, 0xf7, col_s / 4);
+    write_reg(sensor->slv_addr, 0xf8, row_s / 4);
+    write_reg(sensor->slv_addr, 0xf9, (col_s + h) / 4);
+    write_reg(sensor->slv_addr, 0xfa, (row_s + w) / 4);
+
+    write_reg(sensor->slv_addr, 0x05, H8(row_s));
+    write_reg(sensor->slv_addr, 0x06, L8(row_s));
+    write_reg(sensor->slv_addr, 0x07, H8(col_s));
+    write_reg(sensor->slv_addr, 0x08, L8(col_s));
+
+    write_reg(sensor->slv_addr, 0x09, H8(h + 8));
+    write_reg(sensor->slv_addr, 0x0a, L8(h + 8));
+    write_reg(sensor->slv_addr, 0x0b, H8(w + 8));
+    write_reg(sensor->slv_addr, 0x0c, L8(w + 8));
+
+#endif
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set framesize to: %ux%u", w, h);
+    }
+    return 0;
+}
+
+static int set_contrast(sensor_t *sensor, int contrast)
+{
+    if (contrast != 0) {
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        write_reg(sensor->slv_addr, 0xb3, contrast);
+    }
+    return 0;
+}
+
+static int set_global_gain(sensor_t *sensor, int gain_level)
+{
+    if (gain_level != 0) {
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        write_reg(sensor->slv_addr, 0x50, gain_level);
+    }
+    return 0;
+}
+
+static int set_hmirror(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    sensor->status.hmirror = enable;
+    ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    ret |= set_reg_bits(sensor->slv_addr, 0x14, 0, 0x01, enable != 0);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set h-mirror to: %d", enable);
+    }
+    return ret;
+}
+
+static int set_vflip(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    sensor->status.vflip = enable;
+    ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    ret |= set_reg_bits(sensor->slv_addr, 0x14, 1, 0x01, enable != 0);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set v-flip to: %d", enable);
+    }
+    return ret;
+}
+
+static int set_colorbar(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    ret |= set_reg_bits(sensor->slv_addr, 0x2e, 0, 0x01, enable);
+    if (ret == 0) {
+        sensor->status.colorbar = enable;
+        ESP_LOGD(TAG, "Set colorbar to: %d", enable);
+    }
+    return ret;
+}
+
+static int get_reg(sensor_t *sensor, int reg, int mask)
+{
+    int ret = 0;
+    if (mask > 0xFF) {
+        ESP_LOGE(TAG, "mask should not more than 0xff");
+    } else {
+        ret = read_reg(sensor->slv_addr, reg);
+    }
+    if (ret > 0) {
+        ret &= mask;
+    }
+    return ret;
+}
+
+static int set_reg(sensor_t *sensor, int reg, int mask, int value)
+{
+    int ret = 0;
+    if (mask > 0xFF) {
+        ESP_LOGE(TAG, "mask should not more than 0xff");
+    } else {
+        ret = read_reg(sensor->slv_addr, reg);
+    }
+    if (ret < 0) {
+        return ret;
+    }
+    value = (ret & ~mask) | (value & mask);
+
+    if (mask > 0xFF) {
+
+    } else {
+        ret = write_reg(sensor->slv_addr, reg, value);
+    }
+    return ret;
+}
+
+static int init_status(sensor_t *sensor)
+{
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+    sensor->status.brightness = 0;
+    sensor->status.contrast = 0;
+    sensor->status.saturation = 0;
+    sensor->status.sharpness = 0;
+    sensor->status.denoise = 0;
+    sensor->status.ae_level = 0;
+    sensor->status.gainceiling = 0;
+    sensor->status.awb = 0;
+    sensor->status.dcw = 0;
+    sensor->status.agc = 0;
+    sensor->status.aec = 0;
+    sensor->status.hmirror = check_reg_mask(sensor->slv_addr, 0x14, 0x01);
+    sensor->status.vflip = check_reg_mask(sensor->slv_addr, 0x14, 0x02);
+    sensor->status.colorbar = 0;
+    sensor->status.bpc = 0;
+    sensor->status.wpc = 0;
+    sensor->status.raw_gma = 0;
+    sensor->status.lenc = 0;
+    sensor->status.quality = 0;
+    sensor->status.special_effect = 0;
+    sensor->status.wb_mode = 0;
+    sensor->status.awb_gain = 0;
+    sensor->status.agc_gain = 0;
+    sensor->status.aec_value = 0;
+    sensor->status.aec2 = 0;
+
+    print_regs(sensor->slv_addr);
+    return 0;
+}
+
+static int set_dummy(sensor_t *sensor, int val)
+{
+    ESP_LOGW(TAG, "Unsupported");
+    return -1;
+}
+static int set_gainceiling_dummy(sensor_t *sensor, gainceiling_t val)
+{
+    ESP_LOGW(TAG, "Unsupported");
+    return -1;
+}
+
+int gc0308_detect(int slv_addr, sensor_id_t *id)
+{
+    if (GC0308_SCCB_ADDR == slv_addr) {
+        write_reg(slv_addr, 0xfe, 0x00);
+        uint8_t PID = SCCB_Read(slv_addr, 0x00);
+        if (GC0308_PID == PID) {
+            id->PID = PID;
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
+int gc0308_init(sensor_t *sensor)
+{
+    sensor->init_status = init_status;
+    sensor->reset = reset;
+    sensor->set_pixformat = set_pixformat;
+    sensor->set_framesize = set_framesize;
+    sensor->set_contrast = set_contrast;
+    sensor->set_brightness = set_dummy;
+    sensor->set_saturation = set_dummy;
+    sensor->set_sharpness = set_dummy;
+    sensor->set_denoise = set_dummy;
+    sensor->set_gainceiling = set_gainceiling_dummy;
+    sensor->set_quality = set_dummy;
+    sensor->set_colorbar = set_colorbar;
+    sensor->set_whitebal = set_dummy;
+    sensor->set_gain_ctrl = set_global_gain;
+    sensor->set_exposure_ctrl = set_dummy;
+    sensor->set_hmirror = set_hmirror;
+    sensor->set_vflip = set_vflip;
+
+    sensor->set_aec2 = set_dummy;
+    sensor->set_awb_gain = set_dummy;
+    sensor->set_agc_gain = set_dummy;
+    sensor->set_aec_value = set_dummy;
+
+    sensor->set_special_effect = set_dummy;
+    sensor->set_wb_mode = set_dummy;
+    sensor->set_ae_level = set_dummy;
+
+    sensor->set_dcw = set_dummy;
+    sensor->set_bpc = set_dummy;
+    sensor->set_wpc = set_dummy;
+
+    sensor->set_raw_gma = set_dummy;
+    sensor->set_lenc = set_dummy;
+
+    sensor->get_reg = get_reg;
+    sensor->set_reg = set_reg;
+    sensor->set_res_raw = NULL;
+    sensor->set_pll = NULL;
+    sensor->set_xclk = NULL;
+
+    ESP_LOGD(TAG, "GC0308 Attached");
+    return 0;
+}

code/components/esp32-camera-master/sensors/gc032a.c

@@ -0,0 +1,391 @@
+// Copyright 2015-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "sccb.h"
+#include "gc032a.h"
+#include "gc032a_regs.h"
+#include "gc032a_settings.h"
+
+#if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG)
+#include "esp32-hal-log.h"
+#else
+#include "esp_log.h"
+static const char *TAG = "gc032a";
+#endif
+
+#define H8(v) ((v)>>8)
+#define L8(v) ((v)&0xff)
+
+//#define REG_DEBUG_ON
+
+static int read_reg(uint8_t slv_addr, const uint16_t reg)
+{
+    int ret = SCCB_Read(slv_addr, reg);
+#ifdef REG_DEBUG_ON
+    if (ret < 0) {
+        ESP_LOGE(TAG, "READ REG 0x%04x FAILED: %d", reg, ret);
+    }
+#endif
+    return ret;
+}
+
+static int write_reg(uint8_t slv_addr, const uint16_t reg, uint8_t value)
+{
+    int ret = 0;
+#ifndef REG_DEBUG_ON
+    ret = SCCB_Write(slv_addr, reg, value);
+#else
+    int old_value = read_reg(slv_addr, reg);
+    if (old_value < 0) {
+        return old_value;
+    }
+    if ((uint8_t)old_value != value) {
+        ESP_LOGI(TAG, "NEW REG 0x%04x: 0x%02x to 0x%02x", reg, (uint8_t)old_value, value);
+        ret = SCCB_Write(slv_addr, reg, value);
+    } else {
+        ESP_LOGD(TAG, "OLD REG 0x%04x: 0x%02x", reg, (uint8_t)old_value);
+        ret = SCCB_Write(slv_addr, reg, value);//maybe not?
+    }
+    if (ret < 0) {
+        ESP_LOGE(TAG, "WRITE REG 0x%04x FAILED: %d", reg, ret);
+    }
+#endif
+    return ret;
+}
+
+static int check_reg_mask(uint8_t slv_addr, uint16_t reg, uint8_t mask)
+{
+    return (read_reg(slv_addr, reg) & mask) == mask;
+}
+
+static void print_regs(uint8_t slv_addr)
+{
+#ifdef DEBUG_PRINT_REG
+    vTaskDelay(pdMS_TO_TICKS(100));
+    ESP_LOGI(TAG, "REG list look ======================");
+    for (size_t i = 0xf0; i <= 0xfe; i++) {
+        ESP_LOGI(TAG, "reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+    ESP_LOGI(TAG, "\npage 0 ===");
+    write_reg(slv_addr, 0xfe, 0x00); // page 0
+    for (size_t i = 0x03; i <= 0x24; i++) {
+        ESP_LOGI(TAG, "p0 reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+    for (size_t i = 0x40; i <= 0x95; i++) {
+        ESP_LOGI(TAG, "p0 reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+    ESP_LOGI(TAG, "\npage 3 ===");
+    write_reg(slv_addr, 0xfe, 0x03); // page 3
+    for (size_t i = 0x01; i <= 0x43; i++) {
+        ESP_LOGI(TAG, "p3 reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+#endif
+}
+
+static int set_reg_bits(uint8_t slv_addr, uint16_t reg, uint8_t offset, uint8_t mask, uint8_t value)
+{
+    int ret = 0;
+    uint8_t c_value, new_value;
+    ret = read_reg(slv_addr, reg);
+    if (ret < 0) {
+        return ret;
+    }
+    c_value = ret;
+    new_value = (c_value & ~(mask << offset)) | ((value & mask) << offset);
+    ret = write_reg(slv_addr, reg, new_value);
+    return ret;
+}
+
+static int write_regs(uint8_t slv_addr, const uint16_t (*regs)[2])
+{
+    int i = 0, ret = 0;
+    while (!ret && regs[i][0] != REGLIST_TAIL) {
+        if (regs[i][0] == REG_DLY) {
+            vTaskDelay(regs[i][1] / portTICK_PERIOD_MS);
+        } else {
+            ret = write_reg(slv_addr, regs[i][0], regs[i][1]);
+        }
+        i++;
+    }
+    return ret;
+}
+
+static int reset(sensor_t *sensor)
+{
+    int ret;
+    // Software Reset: clear all registers and reset them to their default values
+    ret = write_reg(sensor->slv_addr, RESET_RELATED, 0xf0);
+    if (ret) {
+        ESP_LOGE(TAG, "Software Reset FAILED!");
+        return ret;
+    }
+    vTaskDelay(100 / portTICK_PERIOD_MS);
+
+    ret = write_regs(sensor->slv_addr, gc032a_default_regs);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Camera defaults loaded");
+        vTaskDelay(100 / portTICK_PERIOD_MS);
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        set_reg_bits(sensor->slv_addr, 0xf7, 1, 0x01, 1); // PLL_mode1:div2en
+        set_reg_bits(sensor->slv_addr, 0xf7, 7, 0x01, 1); // PLL_mode1:dvp mode
+        set_reg_bits(sensor->slv_addr, 0xf8, 0, 0x3f, 8); //PLL_mode2 :divx4
+        set_reg_bits(sensor->slv_addr, 0xfa, 4, 0x0f, 2); //vlk div mode :divide_by
+    }
+
+    return ret;
+}
+
+static int set_pixformat(sensor_t *sensor, pixformat_t pixformat)
+{
+    int ret = 0;
+    switch (pixformat) {
+    case PIXFORMAT_RGB565:
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        ret = set_reg_bits(sensor->slv_addr, 0x44, 0, 0x1f, 6);  //RGB565
+        break;
+
+    case PIXFORMAT_YUV422:
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        ret = set_reg_bits(sensor->slv_addr, 0x44, 0, 0x1f, 3);
+        break;
+    default:
+        ESP_LOGW(TAG, "unsupport format");
+        ret = -1;
+        break;
+    }
+    if (ret == 0) {
+        sensor->pixformat = pixformat;
+        ESP_LOGD(TAG, "Set pixformat to: %u", pixformat);
+    }
+
+    return ret;
+}
+
+static int set_framesize(sensor_t *sensor, framesize_t framesize)
+{
+    ESP_LOGI(TAG, "set_framesize");
+    int ret = 0;
+    if (framesize > FRAMESIZE_VGA) {
+        ESP_LOGW(TAG, "Invalid framesize: %u", framesize);
+        framesize = FRAMESIZE_VGA;
+    }
+    sensor->status.framesize = framesize;
+    uint16_t w = resolution[framesize].width;
+    uint16_t h = resolution[framesize].height;
+    uint16_t row_s = (resolution[FRAMESIZE_VGA].height - h) / 2;
+    uint16_t col_s = (resolution[FRAMESIZE_VGA].width - w) / 2;
+
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+    write_reg(sensor->slv_addr, P0_ROW_START_HIGH, H8(row_s)); // Row_start[8]
+    write_reg(sensor->slv_addr, P0_ROW_START_LOW, L8(row_s)); // Row_start[7:0]
+    write_reg(sensor->slv_addr, P0_COLUMN_START_HIGH, H8(col_s)); // Column_start[9:8]
+    write_reg(sensor->slv_addr, P0_COLUMN_START_LOW, L8(col_s)); // Column_start[7:0]
+    write_reg(sensor->slv_addr, P0_WINDOW_HEIGHT_HIGH, H8(h + 8)); //window_height [8]
+    write_reg(sensor->slv_addr, P0_WINDOW_HEIGHT_LOW, L8(h + 8)); //window_height [7:0]
+    write_reg(sensor->slv_addr, P0_WINDOW_WIDTH_HIGH, H8(w + 8)); //window_width [9:8]
+    write_reg(sensor->slv_addr, P0_WINDOW_WIDTH_LOW, L8(w + 8)); //window_width [7:0]
+
+    write_reg(sensor->slv_addr, P0_WIN_MODE, 0x01);
+    write_reg(sensor->slv_addr, P0_OUT_WIN_HEIGHT_HIGH, H8(h));
+    write_reg(sensor->slv_addr, P0_OUT_WIN_HEIGHT_LOW, L8(h));
+    write_reg(sensor->slv_addr, P0_OUT_WIN_WIDTH_HIGH, H8(w));
+    write_reg(sensor->slv_addr, P0_OUT_WIN_WIDTH_LOW, L8(w));
+
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set framesize to: %ux%u", w, h);
+    }
+    print_regs(sensor->slv_addr);
+    return ret;
+}
+
+static int set_hmirror(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    sensor->status.hmirror = enable;
+    ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    ret |= set_reg_bits(sensor->slv_addr, P0_CISCTL_MODE1, 0, 0x01, enable);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set h-mirror to: %d", enable);
+    }
+    return ret;
+}
+
+static int set_vflip(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    sensor->status.vflip = enable;
+    ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    ret |= set_reg_bits(sensor->slv_addr, P0_CISCTL_MODE1, 1, 0x01, enable);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set v-flip to: %d", enable);
+    }
+    return ret;
+}
+
+static int set_colorbar(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    ret |= set_reg_bits(sensor->slv_addr, P0_DEBUG_MODE2, 3, 0x01, enable);
+    if (ret == 0) {
+        sensor->status.colorbar = enable;
+        ESP_LOGD(TAG, "Set colorbar to: %d", enable);
+    }
+    return ret;
+}
+
+static int get_reg(sensor_t *sensor, int reg, int mask)
+{
+    int ret = 0;
+    if (mask > 0xFF) {
+        ESP_LOGE(TAG, "mask should not more than 0xff");
+    } else {
+        ret = read_reg(sensor->slv_addr, reg);
+    }
+    if (ret > 0) {
+        ret &= mask;
+    }
+    return ret;
+}
+
+static int set_reg(sensor_t *sensor, int reg, int mask, int value)
+{
+    int ret = 0;
+    if (mask > 0xFF) {
+        ESP_LOGE(TAG, "mask should not more than 0xff");
+    } else {
+        ret = read_reg(sensor->slv_addr, reg);
+    }
+    if (ret < 0) {
+        return ret;
+    }
+    value = (ret & ~mask) | (value & mask);
+
+    if (mask > 0xFF) {
+
+    } else {
+        ret = write_reg(sensor->slv_addr, reg, value);
+    }
+    return ret;
+}
+
+static int init_status(sensor_t *sensor)
+{
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+    sensor->status.brightness = 0;
+    sensor->status.contrast = 0;
+    sensor->status.saturation = 0;
+    sensor->status.sharpness = 0;
+    sensor->status.denoise = 0;
+    sensor->status.ae_level = 0;
+    sensor->status.gainceiling = 0;
+    sensor->status.awb = 0;
+    sensor->status.dcw = 0;
+    sensor->status.agc = 0;
+    sensor->status.aec = 0;
+    sensor->status.hmirror = check_reg_mask(sensor->slv_addr, P0_CISCTL_MODE1, 0x01);
+    sensor->status.vflip = check_reg_mask(sensor->slv_addr, P0_CISCTL_MODE1, 0x02);
+    sensor->status.colorbar = 0;
+    sensor->status.bpc = 0;
+    sensor->status.wpc = 0;
+    sensor->status.raw_gma = 0;
+    sensor->status.lenc = 0;
+    sensor->status.quality = 0;
+    sensor->status.special_effect = 0;
+    sensor->status.wb_mode = 0;
+    sensor->status.awb_gain = 0;
+    sensor->status.agc_gain = 0;
+    sensor->status.aec_value = 0;
+    sensor->status.aec2 = 0;
+    return 0;
+}
+
+static int set_dummy(sensor_t *sensor, int val)
+{
+    ESP_LOGW(TAG, "Unsupported");
+    return -1;
+}
+static int set_gainceiling_dummy(sensor_t *sensor, gainceiling_t val)
+{
+    ESP_LOGW(TAG, "Unsupported");
+    return -1;
+}
+
+int gc032a_detect(int slv_addr, sensor_id_t *id)
+{
+    if (GC032A_SCCB_ADDR == slv_addr) {
+        uint8_t MIDL = SCCB_Read(slv_addr, SENSOR_ID_LOW);
+        uint8_t MIDH = SCCB_Read(slv_addr, SENSOR_ID_HIGH);
+        uint16_t PID = MIDH << 8 | MIDL;
+        if (GC032A_PID == PID) {
+            id->PID = PID;
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
+int gc032a_init(sensor_t *sensor)
+{
+    sensor->init_status = init_status;
+    sensor->reset = reset;
+    sensor->set_pixformat = set_pixformat;
+    sensor->set_framesize = set_framesize;
+    sensor->set_contrast = set_dummy;
+    sensor->set_brightness = set_dummy;
+    sensor->set_saturation = set_dummy;
+    sensor->set_sharpness = set_dummy;
+    sensor->set_denoise = set_dummy;
+    sensor->set_gainceiling = set_gainceiling_dummy;
+    sensor->set_quality = set_dummy;
+    sensor->set_colorbar = set_colorbar;
+    sensor->set_whitebal = set_dummy;
+    sensor->set_gain_ctrl = set_dummy;
+    sensor->set_exposure_ctrl = set_dummy;
+    sensor->set_hmirror = set_hmirror;
+    sensor->set_vflip = set_vflip;
+
+    sensor->set_aec2 = set_dummy;
+    sensor->set_awb_gain = set_dummy;
+    sensor->set_agc_gain = set_dummy;
+    sensor->set_aec_value = set_dummy;
+
+    sensor->set_special_effect = set_dummy;
+    sensor->set_wb_mode = set_dummy;
+    sensor->set_ae_level = set_dummy;
+
+    sensor->set_dcw = set_dummy;
+    sensor->set_bpc = set_dummy;
+    sensor->set_wpc = set_dummy;
+
+    sensor->set_raw_gma = set_dummy;
+    sensor->set_lenc = set_dummy;
+
+    sensor->get_reg = get_reg;
+    sensor->set_reg = set_reg;
+    sensor->set_res_raw = NULL;
+    sensor->set_pll = NULL;
+    sensor->set_xclk = NULL;
+
+    ESP_LOGD(TAG, "GC032A Attached");
+    return 0;
+}

code/components/esp32-camera-master/sensors/gc2145.c

@@ -0,0 +1,477 @@
+// Copyright 2015-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "sccb.h"
+#include "gc2145.h"
+#include "gc2145_regs.h"
+#include "gc2145_settings.h"
+
+#if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG)
+#include "esp32-hal-log.h"
+#else
+#include "esp_log.h"
+static const char *TAG = "gc2145";
+#endif
+
+#define H8(v) ((v)>>8)
+#define L8(v) ((v)&0xff)
+
+//#define REG_DEBUG_ON
+
+static int read_reg(uint8_t slv_addr, const uint16_t reg)
+{
+    int ret = SCCB_Read(slv_addr, reg);
+#ifdef REG_DEBUG_ON
+    if (ret < 0) {
+        ESP_LOGE(TAG, "READ REG 0x%04x FAILED: %d", reg, ret);
+    }
+#endif
+    return ret;
+}
+
+static int write_reg(uint8_t slv_addr, const uint16_t reg, uint8_t value)
+{
+    int ret = 0;
+#ifndef REG_DEBUG_ON
+    ret = SCCB_Write(slv_addr, reg, value);
+#else
+    int old_value = read_reg(slv_addr, reg);
+    if (old_value < 0) {
+        return old_value;
+    }
+    if ((uint8_t)old_value != value) {
+        ESP_LOGI(TAG, "NEW REG 0x%04x: 0x%02x to 0x%02x", reg, (uint8_t)old_value, value);
+        ret = SCCB_Write(slv_addr, reg, value);
+    } else {
+        ESP_LOGD(TAG, "OLD REG 0x%04x: 0x%02x", reg, (uint8_t)old_value);
+        ret = SCCB_Write(slv_addr, reg, value);//maybe not?
+    }
+    if (ret < 0) {
+        ESP_LOGE(TAG, "WRITE REG 0x%04x FAILED: %d", reg, ret);
+    }
+#endif
+    return ret;
+}
+
+static int check_reg_mask(uint8_t slv_addr, uint16_t reg, uint8_t mask)
+{
+    return (read_reg(slv_addr, reg) & mask) == mask;
+}
+
+static int set_reg_bits(uint8_t slv_addr, uint16_t reg, uint8_t offset, uint8_t mask, uint8_t value)
+{
+    int ret = 0;
+    uint8_t c_value, new_value;
+    ret = read_reg(slv_addr, reg);
+    if (ret < 0) {
+        return ret;
+    }
+    c_value = ret;
+    new_value = (c_value & ~(mask << offset)) | ((value & mask) << offset);
+    ret = write_reg(slv_addr, reg, new_value);
+    return ret;
+}
+
+static int write_regs(uint8_t slv_addr, const uint16_t (*regs)[2])
+{
+    int i = 0, ret = 0;
+    while (!ret && regs[i][0] != REGLIST_TAIL) {
+        if (regs[i][0] == REG_DLY) {
+            vTaskDelay(regs[i][1] / portTICK_PERIOD_MS);
+        } else {
+            ret = write_reg(slv_addr, regs[i][0], regs[i][1]);
+        }
+        i++;
+    }
+    return ret;
+}
+
+static void print_regs(uint8_t slv_addr)
+{
+#ifdef DEBUG_PRINT_REG
+    vTaskDelay(pdMS_TO_TICKS(100));
+    ESP_LOGI(TAG, "REG list look ======================");
+    for (size_t i = 0xf0; i <= 0xfe; i++) {
+        ESP_LOGI(TAG, "reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+    ESP_LOGI(TAG, "\npage 0 ===");
+    write_reg(slv_addr, 0xfe, 0x00); // page 0
+    for (size_t i = 0x03; i <= 0x24; i++) {
+        ESP_LOGI(TAG, "p0 reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+    for (size_t i = 0x80; i <= 0xa2; i++) {
+        ESP_LOGI(TAG, "p0 reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+    ESP_LOGI(TAG, "\npage 3 ===");
+    write_reg(slv_addr, 0xfe, 0x03); // page 3
+    for (size_t i = 0x01; i <= 0x43; i++) {
+        ESP_LOGI(TAG, "p3 reg[0x%02x] = 0x%02x", i, read_reg(slv_addr, i));
+    }
+#endif
+}
+
+static int reset(sensor_t *sensor)
+{
+    int ret = 0;
+    // Software Reset: clear all registers and reset them to their default values
+    ret = write_reg(sensor->slv_addr, RESET_RELATED, 0xe0);
+    if (ret) {
+        ESP_LOGE(TAG, "Software Reset FAILED!");
+        return ret;
+    }
+    vTaskDelay(100 / portTICK_PERIOD_MS);
+    ret = write_regs(sensor->slv_addr, gc2145_default_init_regs);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Camera defaults loaded");
+        vTaskDelay(100 / portTICK_PERIOD_MS);
+#ifdef CONFIG_IDF_TARGET_ESP32
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        //ensure pclk <= 15MHz for esp32
+        set_reg_bits(sensor->slv_addr, 0xf8, 0, 0x3f, 2); // divx4
+        set_reg_bits(sensor->slv_addr, 0xfa, 4, 0x0f, 2); // divide_by
+#endif
+
+    }
+    return ret;
+}
+
+static int set_pixformat(sensor_t *sensor, pixformat_t pixformat)
+{
+    int ret = 0;
+
+    switch (pixformat) {
+    case PIXFORMAT_RGB565:
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        ret = set_reg_bits(sensor->slv_addr, P0_OUTPUT_FORMAT, 0, 0x1f, 6);  //RGB565
+        break;
+
+    case PIXFORMAT_YUV422:
+        write_reg(sensor->slv_addr, 0xfe, 0x00);
+        ret = set_reg_bits(sensor->slv_addr, P0_OUTPUT_FORMAT, 0, 0x1f, 2); //yuv422
+        break;
+    default:
+        ESP_LOGW(TAG, "unsupport format");
+        ret = -1;
+        break;
+    }
+
+    if (ret == 0) {
+        sensor->pixformat = pixformat;
+        ESP_LOGD(TAG, "Set pixformat to: %u", pixformat);
+    }
+    return ret;
+}
+
+static int set_framesize(sensor_t *sensor, framesize_t framesize)
+{
+    int ret = 0;
+    if (framesize > FRAMESIZE_UXGA) {
+        ESP_LOGW(TAG, "Invalid framesize: %u", framesize);
+        framesize = FRAMESIZE_UXGA;
+    }
+    sensor->status.framesize = framesize;
+    uint16_t w = resolution[framesize].width;
+    uint16_t h = resolution[framesize].height;
+    uint16_t row_s = (resolution[FRAMESIZE_UXGA].height - h) / 2;
+    uint16_t col_s = (resolution[FRAMESIZE_UXGA].width - w) / 2;
+    (void)row_s;
+    (void)col_s;
+
+#if CONFIG_GC_SENSOR_SUBSAMPLE_MODE
+    struct subsample_cfg {
+        uint16_t ratio_numerator;
+        uint16_t ratio_denominator;
+        uint8_t reg0x99;
+        uint8_t reg0x9b;
+        uint8_t reg0x9c;
+        uint8_t reg0x9d;
+        uint8_t reg0x9e;
+        uint8_t reg0x9f;
+        uint8_t reg0xa0;
+        uint8_t reg0xa1;
+        uint8_t reg0xa2;
+    };
+    const struct subsample_cfg subsample_cfgs[] = { // define some subsample ratio
+        // {60, 420, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, //1/7 // A smaller ratio brings a larger view, but it reduces the frame rate
+        // {84, 420, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, //1/5
+        // {105, 420, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},//1/4
+        {140, 420, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},//1/3
+        {210, 420, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},//1/2
+        {240, 420, 0x77, 0x02, 0x46, 0x02, 0x46, 0x02, 0x46, 0x02, 0x46},//4/7
+        {252, 420, 0x55, 0x02, 0x04, 0x02, 0x04, 0x02, 0x04, 0x02, 0x04},//3/5
+        {280, 420, 0x33, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00},//2/3
+        {420, 420, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},//1/1
+    };
+    uint16_t win_w = resolution[FRAMESIZE_UXGA].width;
+    uint16_t win_h = resolution[FRAMESIZE_UXGA].height;
+    const struct subsample_cfg *cfg = NULL;
+    /**
+     * Strategy: try to keep the maximum perspective
+     */
+    uint8_t i = 0;
+    if (framesize >= FRAMESIZE_QVGA) {
+        i = 1;
+    }
+    for (; i < sizeof(subsample_cfgs) / sizeof(struct subsample_cfg); i++) {
+        cfg = &subsample_cfgs[i];
+        if ((win_w * cfg->ratio_numerator / cfg->ratio_denominator >= w) && (win_h * cfg->ratio_numerator / cfg->ratio_denominator >= h)) {
+            win_w = w * cfg->ratio_denominator / cfg->ratio_numerator;
+            win_h = h * cfg->ratio_denominator / cfg->ratio_numerator;
+            row_s = (resolution[FRAMESIZE_UXGA].height - win_h) / 2;
+            col_s = (resolution[FRAMESIZE_UXGA].width - win_w) / 2;
+            ESP_LOGI(TAG, "subsample win:%dx%d, ratio:%f", win_w, win_h, (float)cfg->ratio_numerator / (float)cfg->ratio_denominator);
+            break;
+        }
+    }
+
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+    write_reg(sensor->slv_addr, P0_CROP_ENABLE, 0x01);
+    write_reg(sensor->slv_addr, 0x09, H8(row_s));
+    write_reg(sensor->slv_addr, 0x0a, L8(row_s));
+    write_reg(sensor->slv_addr, 0x0b, H8(col_s));
+    write_reg(sensor->slv_addr, 0x0c, L8(col_s));
+    write_reg(sensor->slv_addr, 0x0d, H8(win_h + 8));
+    write_reg(sensor->slv_addr, 0x0e, L8(win_h + 8));
+    write_reg(sensor->slv_addr, 0x0f, H8(win_w + 16));
+    write_reg(sensor->slv_addr, 0x10, L8(win_w + 16));
+
+    write_reg(sensor->slv_addr, 0x99, cfg->reg0x99);
+    write_reg(sensor->slv_addr, 0x9b, cfg->reg0x9b);
+    write_reg(sensor->slv_addr, 0x9c, cfg->reg0x9c);
+    write_reg(sensor->slv_addr, 0x9d, cfg->reg0x9d);
+    write_reg(sensor->slv_addr, 0x9e, cfg->reg0x9e);
+    write_reg(sensor->slv_addr, 0x9f, cfg->reg0x9f);
+    write_reg(sensor->slv_addr, 0xa0, cfg->reg0xa0);
+    write_reg(sensor->slv_addr, 0xa1, cfg->reg0xa1);
+    write_reg(sensor->slv_addr, 0xa2, cfg->reg0xa2);
+
+    write_reg(sensor->slv_addr, 0x95, H8(h));
+    write_reg(sensor->slv_addr, 0x96, L8(h));
+    write_reg(sensor->slv_addr, 0x97, H8(w));
+    write_reg(sensor->slv_addr, 0x98, L8(w));
+
+
+#elif CONFIG_GC_SENSOR_WINDOWING_MODE
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+
+    write_reg(sensor->slv_addr, P0_CROP_ENABLE, 0x01);
+    // write_reg(sensor->slv_addr, 0xec, col_s / 8); //measure window
+    // write_reg(sensor->slv_addr, 0xed, row_s / 8);
+    // write_reg(sensor->slv_addr, 0xee, (col_s + h) / 8);
+    // write_reg(sensor->slv_addr, 0xef, (row_s + w) / 8);
+
+    write_reg(sensor->slv_addr, 0x09, H8(row_s));
+    write_reg(sensor->slv_addr, 0x0a, L8(row_s));
+    write_reg(sensor->slv_addr, 0x0b, H8(col_s));
+    write_reg(sensor->slv_addr, 0x0c, L8(col_s));
+    write_reg(sensor->slv_addr, 0x0d, H8(h + 8));
+    write_reg(sensor->slv_addr, 0x0e, L8(h + 8));
+    write_reg(sensor->slv_addr, 0x0f, H8(w + 8));
+    write_reg(sensor->slv_addr, 0x10, L8(w + 8));
+
+    write_reg(sensor->slv_addr, 0x95, H8(h));
+    write_reg(sensor->slv_addr, 0x96, L8(h));
+    write_reg(sensor->slv_addr, 0x97, H8(w));
+    write_reg(sensor->slv_addr, 0x98, L8(w));
+
+#endif
+
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set framesize to: %ux%u", w, h);
+    }
+    return ret;
+
+}
+
+static int set_hmirror(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    sensor->status.hmirror = enable;
+    ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    ret |= set_reg_bits(sensor->slv_addr, P0_ANALOG_MODE1, 0, 0x01, enable != 0);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set h-mirror to: %d", enable);
+    }
+    return ret;
+}
+
+static int set_vflip(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    sensor->status.vflip = enable;
+    ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    ret |= set_reg_bits(sensor->slv_addr, P0_ANALOG_MODE1, 1, 0x01, enable != 0);
+    if (ret == 0) {
+        ESP_LOGD(TAG, "Set v-flip to: %d", enable);
+    }
+    return ret;
+}
+
+static int set_colorbar(sensor_t *sensor, int enable)
+{
+    int ret = 0;
+    // ret = write_reg(sensor->slv_addr, 0xfe, 0x00);
+    // ret |= set_reg_bits(sensor->slv_addr, P0_DEBUG_MODE3, 3, 0x01, enable);
+    if (ret == 0) {
+        sensor->status.colorbar = enable;
+        ESP_LOGD(TAG, "Set colorbar to: %d", enable);
+    }
+    return ret;
+}
+
+static int get_reg(sensor_t *sensor, int reg, int mask)
+{
+    int ret = 0;
+    if (mask > 0xFF) {
+        ESP_LOGE(TAG, "mask should not more than 0xff");
+    } else {
+        ret = read_reg(sensor->slv_addr, reg);
+    }
+    if (ret > 0) {
+        ret &= mask;
+    }
+    return ret;
+}
+
+static int set_reg(sensor_t *sensor, int reg, int mask, int value)
+{
+    int ret = 0;
+    if (mask > 0xFF) {
+        ESP_LOGE(TAG, "mask should not more than 0xff");
+    } else {
+        ret = read_reg(sensor->slv_addr, reg);
+    }
+    if (ret < 0) {
+        return ret;
+    }
+    value = (ret & ~mask) | (value & mask);
+
+    if (mask > 0xFF) {
+
+    } else {
+        ret = write_reg(sensor->slv_addr, reg, value);
+    }
+    return ret;
+}
+
+static int init_status(sensor_t *sensor)
+{
+    write_reg(sensor->slv_addr, 0xfe, 0x00);
+    sensor->status.brightness = 0;
+    sensor->status.contrast = 0;
+    sensor->status.saturation = 0;
+    sensor->status.sharpness = 0;
+    sensor->status.denoise = 0;
+    sensor->status.ae_level = 0;
+    sensor->status.gainceiling = 0;
+    sensor->status.awb = 0;
+    sensor->status.dcw = 0;
+    sensor->status.agc = 0;
+    sensor->status.aec = 0;
+    sensor->status.hmirror = check_reg_mask(sensor->slv_addr, P0_ANALOG_MODE1, 0x01);
+    sensor->status.vflip = check_reg_mask(sensor->slv_addr, P0_ANALOG_MODE1, 0x02);
+    sensor->status.colorbar = 0;
+    sensor->status.bpc = 0;
+    sensor->status.wpc = 0;
+    sensor->status.raw_gma = 0;
+    sensor->status.lenc = 0;
+    sensor->status.quality = 0;
+    sensor->status.special_effect = 0;
+    sensor->status.wb_mode = 0;
+    sensor->status.awb_gain = 0;
+    sensor->status.agc_gain = 0;
+    sensor->status.aec_value = 0;
+    sensor->status.aec2 = 0;
+
+    print_regs(sensor->slv_addr);
+    return 0;
+}
+
+static int set_dummy(sensor_t *sensor, int val)
+{
+    ESP_LOGW(TAG, "Unsupported");
+    return -1;
+}
+static int set_gainceiling_dummy(sensor_t *sensor, gainceiling_t val)
+{
+    ESP_LOGW(TAG, "Unsupported");
+    return -1;
+}
+
+int gc2145_detect(int slv_addr, sensor_id_t *id)
+{
+    if (GC2145_SCCB_ADDR == slv_addr) {
+        uint8_t MIDL = SCCB_Read(slv_addr, CHIP_ID_LOW);
+        uint8_t MIDH = SCCB_Read(slv_addr, CHIP_ID_HIGH);
+        uint16_t PID = MIDH << 8 | MIDL;
+        if (GC2145_PID == PID) {
+            id->PID = PID;
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
+int gc2145_init(sensor_t *sensor)
+{
+    sensor->init_status = init_status;
+    sensor->reset = reset;
+    sensor->set_pixformat = set_pixformat;
+    sensor->set_framesize = set_framesize;
+    sensor->set_contrast = set_dummy;
+    sensor->set_brightness = set_dummy;
+    sensor->set_saturation = set_dummy;
+    sensor->set_sharpness = set_dummy;
+    sensor->set_denoise = set_dummy;
+    sensor->set_gainceiling = set_gainceiling_dummy;
+    sensor->set_quality = set_dummy;
+    sensor->set_colorbar = set_colorbar;
+    sensor->set_whitebal = set_dummy;
+    sensor->set_gain_ctrl = set_dummy;
+    sensor->set_exposure_ctrl = set_dummy;
+    sensor->set_hmirror = set_hmirror;
+    sensor->set_vflip = set_vflip;
+
+    sensor->set_aec2 = set_dummy;
+    sensor->set_awb_gain = set_dummy;
+    sensor->set_agc_gain = set_dummy;
+    sensor->set_aec_value = set_dummy;
+
+    sensor->set_special_effect = set_dummy;
+    sensor->set_wb_mode = set_dummy;
+    sensor->set_ae_level = set_dummy;
+
+    sensor->set_dcw = set_dummy;
+    sensor->set_bpc = set_dummy;
+    sensor->set_wpc = set_dummy;
+
+    sensor->set_raw_gma = set_dummy;
+    sensor->set_lenc = set_dummy;
+
+    sensor->get_reg = get_reg;
+    sensor->set_reg = set_reg;
+    sensor->set_res_raw = NULL;
+    sensor->set_pll = NULL;
+    sensor->set_xclk = NULL;
+
+    ESP_LOGD(TAG, "GC2145 Attached");
+    return 0;
+}

code/components/esp32-camera-master/sensors/nt99141.c

@@ -10,6 +10,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "sccb.h"
+#include "xclk.h"
 #include "nt99141.h"
 #include "nt99141_regs.h"
 #include "nt99141_settings.h"
@@ -144,28 +145,6 @@ static int write_addr_reg(uint8_t slv_addr, const uint16_t reg, uint16_t x_value
 
 #define write_reg_bits(slv_addr, reg, mask, enable) set_reg_bits(slv_addr, reg, 0, mask, enable?mask:0)
 
-static int calc_sysclk(int xclk, bool pll_bypass, int pll_multiplier, int pll_sys_div, int pll_pre_div, bool pll_root_2x, int pll_seld5, bool pclk_manual, int pclk_div)
-{
-    const int pll_pre_div2x_map[] = { 2, 3, 4, 6 };//values are multiplied by two to avoid floats
-    const int pll_seld52x_map[] = { 2, 2, 4, 5 };
-
-    if (!pll_sys_div) {
-        pll_sys_div = 1;
-    }
-
-    int pll_pre_div2x = pll_pre_div2x_map[pll_pre_div];
-    int pll_root_div = pll_root_2x ? 2 : 1;
-    int pll_seld52x = pll_seld52x_map[pll_seld5];
-
-    int VCO = (xclk / 1000) * pll_multiplier * pll_root_div * 2 / pll_pre_div2x;
-    int PLLCLK = pll_bypass ? (xclk) : (VCO * 1000 * 2 / pll_sys_div / pll_seld52x);
-    int PCLK = PLLCLK / 2 / ((pclk_manual && pclk_div) ? pclk_div : 1);
-    int SYSCLK = PLLCLK / 4;
-
-    ESP_LOGD(TAG, "Calculated VCO: %d Hz, PLLCLK: %d Hz, SYSCLK: %d Hz, PCLK: %d Hz", VCO * 1000, PLLCLK, SYSCLK, PCLK);
-    return SYSCLK;
-}
-
 static int set_pll(sensor_t *sensor, bool bypass, uint8_t multiplier, uint8_t sys_div, uint8_t pre_div, bool root_2x, uint8_t seld5, bool pclk_manual, uint8_t pclk_div)
 {
     return -1;
@@ -309,7 +288,7 @@ static int set_framesize(sensor_t *sensor, framesize_t framesize)
         ret = write_regs(sensor->slv_addr, sensor_framesize_VGA);
     }
 
-    return 0;
+    return ret;
 }
 
 static int set_hmirror(sensor_t *sensor, int enable)
@@ -682,7 +661,6 @@ static int set_brightness(sensor_t *sensor, int level)
 {
     int ret = 0;
     uint8_t value = 0;
-    bool negative = false;
 
     switch (level) {
         case 3:
@@ -699,17 +677,14 @@ static int set_brightness(sensor_t *sensor, int level)
 
         case -1:
             value = 0x78;
-            negative = true;
             break;
 
         case -2:
             value = 0x70;
-            negative = true;
             break;
 
         case -3:
             value = 0x60;
-            negative = true;
             break;
 
         default: // 0
@@ -730,7 +705,6 @@ static int set_contrast(sensor_t *sensor, int level)
 {
     int ret = 0;
     uint8_t value1 = 0, value2 = 0 ;
-    bool negative = false;
 
     switch (level) {
         case 3:
@@ -947,7 +921,6 @@ static int _set_pll(sensor_t *sensor, int bypass, int multiplier, int sys_div, i
     return set_pll(sensor, bypass > 0, multiplier, sys_div, pre_div, root_2x > 0, seld5, pclk_manual > 0, pclk_div);
 }
 
-esp_err_t xclk_timer_conf(int ledc_timer, int xclk_freq_hz);
 static int set_xclk(sensor_t *sensor, int timer, int xclk)
 {
     int ret = 0;
@@ -961,6 +934,23 @@ static int set_xclk(sensor_t *sensor, int timer, int xclk)
     return ret;
 }
 
+int nt99141_detect(int slv_addr, sensor_id_t *id)
+{
+    if (NT99141_SCCB_ADDR == slv_addr) {
+        SCCB_Write16(slv_addr, 0x3008, 0x01);//bank sensor
+        uint16_t h = SCCB_Read16(slv_addr, 0x3000);
+        uint16_t l = SCCB_Read16(slv_addr, 0x3001);
+        uint16_t PID = (h<<8) | l;
+        if (NT99141_PID == PID) {
+            id->PID = PID;
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
 static int init_status(sensor_t *sensor)
 {
     sensor->status.brightness = 0;
@@ -991,7 +981,7 @@ static int init_status(sensor_t *sensor)
     return 0;
 }
 
-int NT99141_init(sensor_t *sensor)
+int nt99141_init(sensor_t *sensor)
 {
     sensor->reset = reset;
     sensor->set_pixformat = set_pixformat;

code/components/esp32-camera-master/sensors/ov2640.c

@@ -10,6 +10,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "sccb.h"
+#include "xclk.h"
 #include "ov2640.h"
 #include "ov2640_regs.h"
 #include "ov2640_settings.h"
@@ -149,7 +150,7 @@ static int set_window(sensor_t *sensor, ov2640_sensor_mode_t mode, int offset_x,
         {VSIZE, max_y & 0xFF},
         {XOFFL, offset_x & 0xFF},
         {YOFFL, offset_y & 0xFF},
-        {VHYX, ((max_y >> 1) & 0X80) | ((offset_y >> 4) & 0X70) | ((max_x >> 5) & 0X08) | ((offset_y >> 8) & 0X07)},
+        {VHYX, ((max_y >> 1) & 0X80) | ((offset_y >> 4) & 0X70) | ((max_x >> 5) & 0X08) | ((offset_x >> 8) & 0X07)},
         {TEST, (max_x >> 2) & 0X80},
         {ZMOW, (w)&0xFF},
         {ZMOH, (h)&0xFF},
@@ -157,26 +158,40 @@ static int set_window(sensor_t *sensor, ov2640_sensor_mode_t mode, int offset_x,
         {0, 0}
     };
 
-    c.pclk_auto = 0;
-    c.pclk_div = 8;
-    c.clk_2x = 0;
-    c.clk_div = 0;
-
-    if(sensor->pixformat != PIXFORMAT_JPEG){
-        c.pclk_auto = 1;
+    if (sensor->pixformat == PIXFORMAT_JPEG) {
+        c.clk_2x = 0;
+        c.clk_div = 0;
+        c.pclk_auto = 0;
+        c.pclk_div = 8;
+        if(mode == OV2640_MODE_UXGA) {
+            c.pclk_div = 12;
+        }
+        // if (sensor->xclk_freq_hz == 16000000) {
+        //     c.pclk_div = c.pclk_div / 2;
+        // }
+    } else {
+#if CONFIG_IDF_TARGET_ESP32
+        c.clk_2x = 0;
+#else
+        c.clk_2x = 1;
+#endif
         c.clk_div = 7;
+        c.pclk_auto = 1;
+        c.pclk_div = 8;
+        if (mode == OV2640_MODE_CIF) {
+            c.clk_div = 3;
+        } else if(mode == OV2640_MODE_UXGA) {
+            c.pclk_div = 12;
+        }
     }
+    ESP_LOGI(TAG, "Set PLL: clk_2x: %u, clk_div: %u, pclk_auto: %u, pclk_div: %u", c.clk_2x, c.clk_div, c.pclk_auto, c.pclk_div);
 
     if (mode == OV2640_MODE_CIF) {
         regs = ov2640_settings_to_cif;
-        if(sensor->pixformat != PIXFORMAT_JPEG){
-            c.clk_div = 3;
-        }
     } else if (mode == OV2640_MODE_SVGA) {
         regs = ov2640_settings_to_svga;
     } else {
         regs = ov2640_settings_to_uxga;
-        c.pclk_div = 12;
     }
 
     WRITE_REG_OR_RETURN(BANK_DSP, R_BYPASS, R_BYPASS_DSP_BYPAS);
@@ -480,7 +495,6 @@ static int _set_pll(sensor_t *sensor, int bypass, int multiplier, int sys_div, i
     return -1;
 }
 
-esp_err_t xclk_timer_conf(int ledc_timer, int xclk_freq_hz);
 static int set_xclk(sensor_t *sensor, int timer, int xclk)
 {
     int ret = 0;
@@ -531,6 +545,24 @@ static int init_status(sensor_t *sensor){
     return 0;
 }
 
+int ov2640_detect(int slv_addr, sensor_id_t *id)
+{
+    if (OV2640_SCCB_ADDR == slv_addr) {
+        SCCB_Write(slv_addr, 0xFF, 0x01);//bank sensor
+        uint16_t PID = SCCB_Read(slv_addr, 0x0A);
+        if (OV2640_PID == PID) {
+            id->PID = PID;
+            id->VER = SCCB_Read(slv_addr, REG_VER);
+            id->MIDL = SCCB_Read(slv_addr, REG_MIDL);
+            id->MIDH = SCCB_Read(slv_addr, REG_MIDH);
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
 int ov2640_init(sensor_t *sensor)
 {
     sensor->reset = reset;

code/components/esp32-camera-master/sensors/ov3660.c

@@ -10,6 +10,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "sccb.h"
+#include "xclk.h"
 #include "ov3660.h"
 #include "ov3660_regs.h"
 #include "ov3660_settings.h"
@@ -142,7 +143,7 @@ static int calc_sysclk(int xclk, bool pll_bypass, int pll_multiplier, int pll_sy
     int PCLK = PLLCLK / 2 / ((pclk_manual && pclk_div)?pclk_div:1);
     int SYSCLK = PLLCLK / 4;
 
-    ESP_LOGD(TAG, "Calculated VCO: %d Hz, PLLCLK: %d Hz, SYSCLK: %d Hz, PCLK: %d Hz", VCO*1000, PLLCLK, SYSCLK, PCLK);
+    ESP_LOGI(TAG, "Calculated VCO: %d Hz, PLLCLK: %d Hz, SYSCLK: %d Hz, PCLK: %d Hz", VCO*1000, PLLCLK, SYSCLK, PCLK);
     return SYSCLK;
 }
 
@@ -310,13 +311,13 @@ static int set_image_options(sensor_t *sensor)
 static int set_framesize(sensor_t *sensor, framesize_t framesize)
 {
     int ret = 0;
-    framesize_t old_framesize = sensor->status.framesize;
-    sensor->status.framesize = framesize;
 
     if(framesize > FRAMESIZE_QXGA){
-        ESP_LOGE(TAG, "Invalid framesize: %u", framesize);
-        return -1;
+        ESP_LOGW(TAG, "Invalid framesize: %u", framesize);
+        framesize = FRAMESIZE_QXGA;
     }
+    framesize_t old_framesize = sensor->status.framesize;
+    sensor->status.framesize = framesize;
     uint16_t w = resolution[framesize].width;
     uint16_t h = resolution[framesize].height;
     aspect_ratio_t ratio = resolution[sensor->status.framesize].aspect_ratio;
@@ -355,7 +356,7 @@ static int set_framesize(sensor_t *sensor, framesize_t framesize)
     }
 
     if (sensor->pixformat == PIXFORMAT_JPEG) {
-        if (framesize == FRAMESIZE_QXGA) {
+        if (framesize == FRAMESIZE_QXGA || sensor->xclk_freq_hz == 16000000) {
             //40MHz SYSCLK and 10MHz PCLK
             ret = set_pll(sensor, false, 24, 1, 3, false, 0, true, 8);
         } else {
@@ -363,12 +364,16 @@ static int set_framesize(sensor_t *sensor, framesize_t framesize)
             ret = set_pll(sensor, false, 30, 1, 3, false, 0, true, 10);
         }
     } else {
-        if (framesize > FRAMESIZE_CIF) {
-            //10MHz SYSCLK and 10MHz PCLK (6.19 FPS)
-            ret = set_pll(sensor, false, 2, 1, 0, false, 0, true, 2);
+        //tuned for 16MHz XCLK and 8MHz PCLK
+        if (framesize > FRAMESIZE_HVGA) {
+            //8MHz SYSCLK and 8MHz PCLK (4.44 FPS)
+            ret = set_pll(sensor, false, 4, 1, 0, false, 2, true, 2);
+        } else if (framesize >= FRAMESIZE_QVGA) {
+            //16MHz SYSCLK and 8MHz PCLK (10.25 FPS)
+            ret = set_pll(sensor, false, 8, 1, 0, false, 2, true, 4);
         } else {
-            //25MHz SYSCLK and 10MHz PCLK (15.45 FPS)
-            ret = set_pll(sensor, false, 5, 1, 0, false, 0, true, 5);
+            //32MHz SYSCLK and 8MHz PCLK (17.77 FPS)
+            ret = set_pll(sensor, false, 8, 1, 0, false, 0, true, 8);
         }
     }
 
@@ -953,7 +958,6 @@ static int _set_pll(sensor_t *sensor, int bypass, int multiplier, int sys_div, i
     return set_pll(sensor, bypass > 0, multiplier, sys_div, pre_div, root_2x > 0, seld5, pclk_manual > 0, pclk_div);
 }
 
-esp_err_t xclk_timer_conf(int ledc_timer, int xclk_freq_hz);
 static int set_xclk(sensor_t *sensor, int timer, int xclk)
 {
     int ret = 0;
@@ -992,6 +996,22 @@ static int init_status(sensor_t *sensor)
     return 0;
 }
 
+int ov3660_detect(int slv_addr, sensor_id_t *id)
+{
+    if (OV3660_SCCB_ADDR == slv_addr) {
+        uint8_t h = SCCB_Read16(slv_addr, 0x300A);
+        uint8_t l = SCCB_Read16(slv_addr, 0x300B);
+        uint16_t PID = (h<<8) | l;
+        if (OV3660_PID == PID) {
+            id->PID = PID;
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
 int ov3660_init(sensor_t *sensor)
 {
     sensor->reset = reset;

code/components/esp32-camera-master/sensors/ov5640.c

@@ -10,6 +10,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "sccb.h"
+#include "xclk.h"
 #include "ov5640.h"
 #include "ov5640_regs.h"
 #include "ov5640_settings.h"
@@ -196,7 +197,7 @@ static int calc_sysclk(int xclk, bool pll_bypass, int pll_multiplier, int pll_sy
 
     unsigned int SYSCLK = PLL_CLK / 4;
 
-    ESP_LOGD(TAG, "Calculated XVCLK: %d Hz, REFIN: %u Hz, VCO: %u Hz, PLL_CLK: %u Hz, SYSCLK: %u Hz, PCLK: %u Hz", xclk, REFIN, VCO, PLL_CLK, SYSCLK, PCLK);
+    ESP_LOGI(TAG, "Calculated XVCLK: %d Hz, REFIN: %u Hz, VCO: %u Hz, PLL_CLK: %u Hz, SYSCLK: %u Hz, PCLK: %u Hz", xclk, REFIN, VCO, PLL_CLK, SYSCLK, PCLK);
     return SYSCLK;
 }
 
@@ -209,6 +210,7 @@ static int set_pll(sensor_t *sensor, bool bypass, uint8_t multiplier, uint8_t sy
     if(multiplier > 127){
         multiplier &= 0xFE;//only even integers above 127
     }
+    ESP_LOGI(TAG, "Set PLL: bypass: %u, multiplier: %u, sys_div: %u, pre_div: %u, root_2x: %u, pclk_root_div: %u, pclk_manual: %u, pclk_div: %u", bypass, multiplier, sys_div, pre_div, root_2x, pclk_root_div, pclk_manual, pclk_div);
 
     calc_sysclk(sensor->xclk_freq_hz, bypass, multiplier, sys_div, pre_div, root_2x, pclk_root_div, pclk_manual, pclk_div);
 
@@ -432,14 +434,22 @@ static int set_framesize(sensor_t *sensor, framesize_t framesize)
     if (sensor->pixformat == PIXFORMAT_JPEG) {
         //10MHz PCLK
         uint8_t sys_mul = 200;
-        if(framesize < FRAMESIZE_QVGA){
+        if(framesize < FRAMESIZE_QVGA || sensor->xclk_freq_hz == 16000000){
             sys_mul = 160;
         } else if(framesize < FRAMESIZE_XGA){
             sys_mul = 180;
         }
         ret = set_pll(sensor, false, sys_mul, 4, 2, false, 2, true, 4);
+        //Set PLL: bypass: 0, multiplier: sys_mul, sys_div: 4, pre_div: 2, root_2x: 0, pclk_root_div: 2, pclk_manual: 1, pclk_div: 4
     } else {
-        ret = set_pll(sensor, false, 10, 1, 1, false, 1, true, 4);
+        //ret = set_pll(sensor, false, 8, 1, 1, false, 1, true, 4);
+        if (framesize > FRAMESIZE_HVGA) {
+            ret = set_pll(sensor, false, 10, 1, 2, false, 1, true, 2);
+        } else if (framesize >= FRAMESIZE_QVGA) {
+            ret = set_pll(sensor, false, 8, 1, 1, false, 1, true, 4);
+        } else {
+            ret = set_pll(sensor, false, 20, 1, 1, false, 1, true, 8);
+        }
     }
 
     if (ret == 0) {
@@ -1025,7 +1035,6 @@ static int _set_pll(sensor_t *sensor, int bypass, int multiplier, int sys_div, i
     return ret;
 }
 
-esp_err_t xclk_timer_conf(int ledc_timer, int xclk_freq_hz);
 static int set_xclk(sensor_t *sensor, int timer, int xclk)
 {
     int ret = 0;
@@ -1064,6 +1073,22 @@ static int init_status(sensor_t *sensor)
     return 0;
 }
 
+int ov5640_detect(int slv_addr, sensor_id_t *id)
+{
+    if (OV5640_SCCB_ADDR == slv_addr) {
+        uint8_t h = SCCB_Read16(slv_addr, 0x300A);
+        uint8_t l = SCCB_Read16(slv_addr, 0x300B);
+        uint16_t PID = (h<<8) | l;
+        if (OV5640_PID == PID) {
+            id->PID = PID;
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
 int ov5640_init(sensor_t *sensor)
 {
     sensor->reset = reset;

code/components/esp32-camera-master/sensors/ov7670.c

@@ -45,7 +45,7 @@ static struct regval_list ov7670_default_regs[] = {
 	{CLKRC, 0x00},  
 	{DBLV,  0x4A},  
 
-    {COM10, COM10_VSYNC_NEG | COM10_PCLK_MASK},
+    {COM10, COM10_VSYNC_NEG | COM10_PCLK_FREE},
 
     /* Improve white balance */ 
 	{COM4, 0x40},  
@@ -393,6 +393,24 @@ static int init_status(sensor_t *sensor)
 static int set_dummy(sensor_t *sensor, int val){ return -1; }
 static int set_gainceiling_dummy(sensor_t *sensor, gainceiling_t val){ return -1; }
 
+int ov7670_detect(int slv_addr, sensor_id_t *id)
+{
+    if (OV7670_SCCB_ADDR == slv_addr) {
+        SCCB_Write(slv_addr, 0xFF, 0x01);//bank sensor
+        uint16_t PID = SCCB_Read(slv_addr, 0x0A);
+        if (OV7670_PID == PID) {
+            id->PID = PID;
+            id->VER = SCCB_Read(slv_addr, REG_VER);
+            id->MIDL = SCCB_Read(slv_addr, REG_MIDL);
+            id->MIDH = SCCB_Read(slv_addr, REG_MIDH);
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
 int ov7670_init(sensor_t *sensor)
 {
     // Set function pointers

code/components/esp32-camera-master/sensors/ov7725.c

@@ -11,6 +11,7 @@
 #include <string.h>
 #include <stdio.h>
 #include "sccb.h"
+#include "xclk.h"
 #include "ov7725.h"
 #include "ov7725_regs.h"
 #include "freertos/FreeRTOS.h"
@@ -58,10 +59,10 @@ static const uint8_t default_regs[][2] = {
     {COM8,          0xF0},
     {COM6,          0xC5},
     {COM9,          0x11},
-    {COM10,         COM10_VSYNC_NEG | COM10_PCLK_MASK}, //Invert VSYNC and MASK PCLK
+    {COM10,         COM10_VSYNC_NEG | COM10_PCLK_FREE}, //Invert VSYNC and MASK PCLK
     {BDBASE,        0x7F},
     {DBSTEP,        0x03},
-    {AEW,           0x96},
+    {AEW,           0x75},
     {AEB,           0x64},
     {VPT,           0xA1},
     {EXHCL,         0x00},
@@ -493,7 +494,6 @@ static int set_gainceiling_dummy(sensor_t *sensor, gainceiling_t val){ return -1
 static int set_res_raw(sensor_t *sensor, int startX, int startY, int endX, int endY, int offsetX, int offsetY, int totalX, int totalY, int outputX, int outputY, bool scale, bool binning){return -1;}
 static int _set_pll(sensor_t *sensor, int bypass, int multiplier, int sys_div, int root_2x, int pre_div, int seld5, int pclk_manual, int pclk_div){return -1;}
 
-esp_err_t xclk_timer_conf(int ledc_timer, int xclk_freq_hz);
 static int set_xclk(sensor_t *sensor, int timer, int xclk)
 {
     int ret = 0;
@@ -502,6 +502,24 @@ static int set_xclk(sensor_t *sensor, int timer, int xclk)
     return ret;
 }
 
+int ov7725_detect(int slv_addr, sensor_id_t *id)
+{
+    if (OV7725_SCCB_ADDR == slv_addr) {
+        SCCB_Write(slv_addr, 0xFF, 0x01);//bank sensor
+        uint16_t PID = SCCB_Read(slv_addr, 0x0A);
+        if (OV7725_PID == PID) {
+            id->PID = PID;
+            id->VER = SCCB_Read(slv_addr, REG_VER);
+            id->MIDL = SCCB_Read(slv_addr, REG_MIDL);
+            id->MIDH = SCCB_Read(slv_addr, REG_MIDH);
+            return PID;
+        } else {
+            ESP_LOGI(TAG, "Mismatch PID=0x%x", PID);
+        }
+    }
+    return 0;
+}
+
 int ov7725_init(sensor_t *sensor)
 {
     // Set function pointers

code/components/esp32-camera-master/sensors/private_include/bf3005.h

@@ -0,0 +1,33 @@
+/*
+ * This file is part of the OpenMV project.
+ * Copyright (c) 2013/2014 Ibrahim Abdelkader <i.abdalkader@gmail.com>
+ * This work is licensed under the MIT license, see the file LICENSE for details.
+ *
+ * BF3005 driver.
+ *
+ */
+#ifndef __BF3005_H__
+#define __BF3005_H__
+#include "sensor.h"
+
+/**
+ * @brief Detect sensor pid
+ *
+ * @param slv_addr SCCB address
+ * @param id Detection result
+ * @return
+ *     0:       Can't detect this sensor
+ *     Nonzero: This sensor has been detected
+ */
+int bf3005_detect(int slv_addr, sensor_id_t *id);
+
+/**
+ * @brief initialize sensor function pointers
+ *
+ * @param sensor pointer of sensor
+ * @return
+ *      Always 0
+ */
+int bf3005_init(sensor_t *sensor);
+
+#endif // __BF3005_H__