add Webhook #3148 (#3163)

* WIP add Webhook

* fix config html for webhook
add tooltips for webhook

* webhook: fix not enabling webhook

* send webhook as json

* Update ApiKey.md

* webhook: fix only sending last "Number"

* webhook JSON is now closer to the data log in CSV format

* webhook: drop timeStampTimeUTC and switch from timeStamp to lastvalue like lokal csv to fix no timestamp on error

---------

Co-authored-by: CaCO3 <caco3@ruinelli.ch>

code/components/jomjol_flowcontroll/CMakeLists.txt

@@ -2,6 +2,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*)
 
 idf_component_register(SRCS ${app_sources}
                     INCLUDE_DIRS "."
-                    REQUIRES esp_timer esp_wifi jomjol_tfliteclass jomjol_helper jomjol_controlcamera jomjol_mqtt jomjol_influxdb jomjol_fileserver_ota jomjol_image_proc jomjol_wlan openmetrics)
+                    REQUIRES esp_timer esp_wifi jomjol_tfliteclass jomjol_helper jomjol_controlcamera jomjol_mqtt jomjol_influxdb jomjol_webhook jomjol_fileserver_ota jomjol_image_proc jomjol_wlan openmetrics)
 
 

code/components/jomjol_flowcontroll/ClassFlowControll.cpp

@@ -64,6 +64,11 @@ std::string ClassFlowControll::doSingleStep(std::string _stepname, std::string _
         _classname = "ClassFlowInfluxDBv2";
     }
     #endif //ENABLE_INFLUXDB
+    #ifdef ENABLE_WEBHOOK
+    if ((_stepname.compare("[Webhook]") == 0) || (_stepname.compare(";[Webhook]") == 0)){
+        _classname = "ClassFlowWebhook";
+    }
+    #endif //ENABLE_WEBHOOK
 
     for (int i = 0; i < FlowControll.size(); ++i)
         if (FlowControll[i]->name().compare(_classname) == 0){
@@ -109,7 +114,11 @@ std::string ClassFlowControll::TranslateAktstatus(std::string _input)
             return ("Sending InfluxDBv2");
         }
     #endif //ENABLE_INFLUXDB
-		
+    #ifdef ENABLE_WEBHOOK
+    if (_input.compare("ClassFlowWebhook") == 0) {
+        return ("Sending Webhook");
+    }
+    #endif //ENABLE_WEBHOOK
     if (_input.compare("ClassFlowPostProcessing") == 0) {
         return ("Post-Processing");
     }
@@ -251,6 +260,10 @@ ClassFlow* ClassFlowControll::CreateClassFlow(std::string _type)
         cfc = new ClassFlowInfluxDBv2(&FlowControll);
     }
     #endif //ENABLE_INFLUXDB  
+    #ifdef ENABLE_WEBHOOK
+    if (toUpper(_type).compare("[WEBHOOK]") == 0)
+        cfc = new ClassFlowWebhook(&FlowControll);
+    #endif //ENABLE_WEBHOOK
 
     if (toUpper(_type).compare("[POSTPROCESSING]") == 0) {
         cfc = new ClassFlowPostProcessing(&FlowControll, flowanalog, flowdigit); 

code/components/jomjol_flowcontroll/ClassFlowControll.h

@@ -17,6 +17,9 @@
 	#include "ClassFlowInfluxDB.h"
 	#include "ClassFlowInfluxDBv2.h"
 #endif //ENABLE_INFLUXDB
+#ifdef ENABLE_WEBHOOK
+	#include "ClassFlowWebhook.h"
+#endif //ENABLE_WEBHOOK
 #include "ClassFlowCNNGeneral.h"
 
 class ClassFlowControll :

code/components/jomjol_flowcontroll/ClassFlowWebhook.cpp

@@ -0,0 +1,143 @@
+#ifdef ENABLE_WEBHOOK
+#include <sstream>
+#include "ClassFlowWebhook.h"
+#include "Helper.h"
+#include "connect_wlan.h"
+
+#include "time_sntp.h"
+#include "interface_webhook.h"
+
+#include "ClassFlowPostProcessing.h"
+#include "esp_log.h"
+#include "../../include/defines.h"
+
+#include "ClassLogFile.h"
+
+#include <time.h>
+
+static const char* TAG = "WEBHOOK";
+
+void ClassFlowWebhook::SetInitialParameter(void)
+{
+    uri = "";
+    flowpostprocessing = NULL;  
+    previousElement = NULL;
+    ListFlowControll = NULL; 
+    disabled = false;
+    WebhookEnable = false;
+}       
+
+ClassFlowWebhook::ClassFlowWebhook()
+{
+    SetInitialParameter();
+}
+
+ClassFlowWebhook::ClassFlowWebhook(std::vector<ClassFlow*>* lfc)
+{
+    SetInitialParameter();
+
+    ListFlowControll = lfc;
+    for (int i = 0; i < ListFlowControll->size(); ++i)
+    {
+        if (((*ListFlowControll)[i])->name().compare("ClassFlowPostProcessing") == 0)
+        {
+            flowpostprocessing = (ClassFlowPostProcessing*) (*ListFlowControll)[i];
+        }
+    }
+}
+
+ClassFlowWebhook::ClassFlowWebhook(std::vector<ClassFlow*>* lfc, ClassFlow *_prev)
+{
+    SetInitialParameter();
+
+    previousElement = _prev;
+    ListFlowControll = lfc;
+
+    for (int i = 0; i < ListFlowControll->size(); ++i)
+    {
+        if (((*ListFlowControll)[i])->name().compare("ClassFlowPostProcessing") == 0)
+        {
+            flowpostprocessing = (ClassFlowPostProcessing*) (*ListFlowControll)[i];
+        }
+    }
+}
+
+
+bool ClassFlowWebhook::ReadParameter(FILE* pfile, string& aktparamgraph)
+{
+    std::vector<string> splitted;
+
+    aktparamgraph = trim(aktparamgraph);
+    printf("akt param: %s\n", aktparamgraph.c_str());
+
+    if (aktparamgraph.size() == 0)
+        if (!this->GetNextParagraph(pfile, aktparamgraph))
+            return false;
+
+    if (toUpper(aktparamgraph).compare("[WEBHOOK]") != 0) 
+        return false;
+
+    
+
+    while (this->getNextLine(pfile, &aktparamgraph) && !this->isNewParagraph(aktparamgraph))
+    {
+        ESP_LOGD(TAG, "while loop reading line: %s", aktparamgraph.c_str());
+        splitted = ZerlegeZeile(aktparamgraph);
+        std::string _param = GetParameterName(splitted[0]);
+            
+        if ((toUpper(_param) == "URI") && (splitted.size() > 1))
+        {
+            this->uri = splitted[1];
+        }
+        if (((toUpper(_param) == "APIKEY")) && (splitted.size() > 1))
+        {
+            this->apikey = splitted[1];
+        }
+    }
+    
+    WebhookInit(uri,apikey);
+    WebhookEnable = true;
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Webhook Enabled for Uri " + uri);
+
+    printf("uri:         %s\n", uri.c_str());   
+    return true;
+}
+
+
+void ClassFlowWebhook::handleMeasurement(string _decsep, string _value)
+{
+    string _digit, _decpos;
+    int _pospunkt = _decsep.find_first_of(".");
+//    ESP_LOGD(TAG, "Name: %s, Pospunkt: %d", _decsep.c_str(), _pospunkt);
+    if (_pospunkt > -1)
+        _digit = _decsep.substr(0, _pospunkt);
+    else
+        _digit = "default";
+    for (int j = 0; j < flowpostprocessing->NUMBERS.size(); ++j)
+    {
+        if (_digit == "default")                        //  Set to default first (if nothing else is set)
+        {
+            flowpostprocessing->NUMBERS[j]->MeasurementV2 = _value;
+        }
+        if (flowpostprocessing->NUMBERS[j]->name == _digit)
+        {
+            flowpostprocessing->NUMBERS[j]->MeasurementV2 = _value;
+        }
+    }
+}
+
+
+bool ClassFlowWebhook::doFlow(string zwtime)
+{
+    if (!WebhookEnable)
+        return true;
+
+    if (flowpostprocessing)
+    {
+        printf("vor sende WebHook");
+        WebhookPublish(flowpostprocessing->GetNumbers());
+    }
+       
+    return true;
+}
+#endif //ENABLE_WEBHOOK

code/components/jomjol_flowcontroll/ClassFlowWebhook.h

@@ -0,0 +1,39 @@
+#ifdef ENABLE_WEBHOOK
+
+#pragma once
+
+#ifndef CLASSFWEBHOOK_H
+#define CLASSFWEBHOOK_H
+
+#include "ClassFlow.h"
+
+#include "ClassFlowPostProcessing.h"
+
+#include <string>
+
+class ClassFlowWebhook :
+    public ClassFlow
+{
+protected:
+    std::string uri, apikey;
+	ClassFlowPostProcessing* flowpostprocessing;  
+    bool WebhookEnable;
+
+    void SetInitialParameter(void); 
+
+    void handleFieldname(string _decsep, string _value);   
+    void handleMeasurement(string _decsep, string _value);
+
+
+public:
+    ClassFlowWebhook();
+    ClassFlowWebhook(std::vector<ClassFlow*>* lfc);
+    ClassFlowWebhook(std::vector<ClassFlow*>* lfc, ClassFlow *_prev);
+
+    bool ReadParameter(FILE* pfile, string& aktparamgraph);
+    bool doFlow(string time);
+    string name(){return "ClassFlowWebhook";};
+};
+
+#endif //CLASSFWEBHOOK_H
+#endif //ENABLE_WEBHOOK

code/components/jomjol_webhook/CMakeLists.txt

@@ -0,0 +1,7 @@
+FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*)
+
+idf_component_register(SRCS ${app_sources}
+                    INCLUDE_DIRS "."
+                    REQUIRES esp_http_client jomjol_logfile jomjol_flowcontroll json)
+
+

code/components/jomjol_webhook/interface_webhook.cpp

@@ -0,0 +1,128 @@
+#ifdef ENABLE_WEBHOOK
+#include "interface_webhook.h"
+
+#include "esp_log.h"
+#include <time.h>
+#include "ClassLogFile.h"
+#include "esp_http_client.h"
+#include "time_sntp.h"
+#include "../../include/defines.h"
+#include <cJSON.h>
+#include <ClassFlowDefineTypes.h>
+
+
+static const char *TAG = "WEBHOOK";
+
+std::string _webhookURI;
+std::string _webhookApiKey;
+
+static esp_err_t http_event_handler(esp_http_client_event_t *evt);
+
+void WebhookInit(std::string _uri, std::string _apiKey)
+{
+    _webhookURI = _uri;
+    _webhookApiKey = _apiKey;
+}
+
+void WebhookPublish(std::vector<NumberPost*>* numbers)
+{
+    
+
+    
+
+    cJSON *jsonArray = cJSON_CreateArray();
+
+    for (int i = 0; i < (*numbers).size(); ++i)
+    {
+        string timezw = "";
+        char buffer[80];
+        struct tm* timeinfo = localtime(&(*numbers)[i]->lastvalue);
+        strftime(buffer, 80, PREVALUE_TIME_FORMAT_OUTPUT, timeinfo);
+        timezw = std::string(buffer);
+
+        cJSON *json = cJSON_CreateObject();
+        cJSON_AddStringToObject(json, "timestamp", timezw.c_str());
+        cJSON_AddStringToObject(json, "name", (*numbers)[i]->name.c_str());
+        cJSON_AddStringToObject(json, "rawValue", (*numbers)[i]->ReturnRawValue.c_str());
+        cJSON_AddStringToObject(json, "value", (*numbers)[i]->ReturnValue.c_str());
+        cJSON_AddStringToObject(json, "preValue", (*numbers)[i]->ReturnPreValue.c_str());
+        cJSON_AddStringToObject(json, "rate", (*numbers)[i]->ReturnRateValue.c_str());
+        cJSON_AddStringToObject(json, "changeAbsolute", (*numbers)[i]->ReturnChangeAbsolute.c_str());
+        cJSON_AddStringToObject(json, "error", (*numbers)[i]->ErrorMessageText.c_str());
+        
+        cJSON_AddItemToArray(jsonArray, json);
+    }
+
+    char *jsonString = cJSON_PrintUnformatted(jsonArray);
+
+    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "sending webhook");
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "sending JSON: " + std::string(jsonString));
+
+    char response_buffer[MAX_HTTP_OUTPUT_BUFFER] = {0};
+    esp_http_client_config_t http_config = {
+        .url = _webhookURI.c_str(),
+        .user_agent = "ESP32 Meter reader",
+        .method = HTTP_METHOD_POST,
+        .event_handler = http_event_handler,
+        .buffer_size = MAX_HTTP_OUTPUT_BUFFER,
+        .user_data = response_buffer
+    };
+
+    esp_http_client_handle_t http_client = esp_http_client_init(&http_config);
+
+    esp_http_client_set_header(http_client, "Content-Type", "application/json");
+    esp_http_client_set_header(http_client, "APIKEY", _webhookApiKey.c_str());
+
+    ESP_ERROR_CHECK(esp_http_client_set_post_field(http_client, jsonString, strlen(jsonString)));
+
+    esp_err_t err = ESP_ERROR_CHECK_WITHOUT_ABORT(esp_http_client_perform(http_client));
+
+    if(err == ESP_OK) {
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP request was performed");
+        int status_code = esp_http_client_get_status_code(http_client);
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP status code: " + std::to_string(status_code));
+    } else {
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "HTTP request failed");
+    }
+
+    
+
+    esp_http_client_cleanup(http_client);
+    cJSON_Delete(jsonArray);
+    free(jsonString);
+}
+
+static esp_err_t http_event_handler(esp_http_client_event_t *evt)
+{
+    switch(evt->event_id)
+    {
+        case HTTP_EVENT_ERROR:
+            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP Client Error encountered");
+            break;
+        case HTTP_EVENT_ON_CONNECTED:
+            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP Client connected");
+            ESP_LOGI(TAG, "HTTP Client Connected");
+            break;
+        case HTTP_EVENT_HEADERS_SENT:
+            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP Client sent all request headers");
+            break;
+        case HTTP_EVENT_ON_HEADER:
+            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Header: key=" + std::string(evt->header_key) + ", value="  + std::string(evt->header_value));
+            break;
+        case HTTP_EVENT_ON_DATA:
+            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP Client data recevied: len=" + std::to_string(evt->data_len));
+            break;
+        case HTTP_EVENT_ON_FINISH:
+            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP Client finished");
+            break;
+         case HTTP_EVENT_DISCONNECTED:
+            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP Client Disconnected");
+            break;
+        case HTTP_EVENT_REDIRECT:
+            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP Redirect");
+            break;
+    }
+    return ESP_OK;
+}
+
+#endif //ENABLE_WEBHOOK

code/components/jomjol_webhook/interface_webhook.h

@@ -0,0 +1,16 @@
+#ifdef ENABLE_WEBHOOK
+
+#pragma once
+#ifndef INTERFACE_WEBHOOK_H
+#define INTERFACE_WEBHOOK_H
+
+#include <string>
+#include <map>
+#include <functional>
+#include <ClassFlowDefineTypes.h>
+
+void WebhookInit(std::string _webhookURI, std::string _apiKey);
+void WebhookPublish(std::vector<NumberPost*>* numbers);
+
+#endif //INTERFACE_WEBHOOK_H
+#endif //ENABLE_WEBHOOK

code/platformio.ini

@@ -58,7 +58,8 @@ build_flags =
 	${flags:runtime.build_flags}
     ; ### Sofware options : (can be set in defines.h)
 	-D ENABLE_MQTT 
-    -D ENABLE_INFLUXDB 
+    -D ENABLE_INFLUXDB
+    -D ENABLE_WEBHOOK
     -D ENABLE_SOFTAP 
 board_build.partitions = partitions.csv
 monitor_speed = 115200
@@ -79,7 +80,8 @@ build_flags =
 	${flags:clangtidy.build_flags}
     ; ### Sofware options : (can be set in defines.h)
 	-D ENABLE_MQTT 
-    -D ENABLE_INFLUXDB 
+    -D ENABLE_INFLUXDB
+    -D ENABLE_WEBHOOK
     ;-D ENABLE_SOFTAP 
     ; ### Debug options :
     ;-D DEBUG_DETAIL_ON
@@ -211,5 +213,6 @@ build_flags =
 	${flags:clangtidy.build_flags}
     ; ### Sofware options :
 	-D ENABLE_MQTT 
-    -D ENABLE_INFLUXDB 
+    -D ENABLE_INFLUXDB
+    -D ENABLE_WEBHOOK
     ;-D ENABLE_SOFTAP ; disabled

param-docs/parameter-pages/Webhook/ApiKey.md

@@ -0,0 +1,4 @@
+# Parameter `ApiKey`
+Default Value: `undefined`
+
+ApiKey sent as Header

param-docs/parameter-pages/Webhook/Uri.md

@@ -0,0 +1,4 @@
+# Parameter `Uri`
+Default Value: `undefined`
+
+URI of the HTTP Endpoint receiving requests, e.g. `http://192.168.1.1/watermeter/webhook`.

sd-card/config/config.ini

@@ -103,6 +103,10 @@ HomeassistantDiscovery = false
 ;Token = undefined
 ;main.Fieldname = undefined
 
+;[Webhook]
+;Uri = undefined
+;ApiKey = undefined
+
 ;[GPIO]
 ;MainTopicMQTT = wasserzaehler/GPIO
 ;IO0 = input disabled 10 false false 

sd-card/html/edit_config_template.html

@@ -1324,6 +1324,37 @@
 			<td>$TOOLTIP_InfluxDBv2_NUMBER.Field</td>
 		</tr>
 
+		<!------------- Webhook ------------------>
+		<tr style="border-bottom: 2px solid lightgray;">
+			<td colspan="3" style="padding-left: 0px; padding-bottom: 3px;">
+			    <h4>
+				<input type="checkbox" id="Category_Webhook_enabled" value="1"  onclick = 'UpdateAfterCategoryCheck()' unchecked >
+				<label for=Category_Webhook_enabled>Webhook</label></h4>		
+			</td>
+		</tr>
+
+		<tr class="WebhookItem">
+			<td class="indent1">
+				<input type="checkbox" id="Webhook_Uri_enabled" value="1"  onclick = 'InvertEnableItem("Webhook", "Uri")' unchecked >
+				<label for=Webhook_Uri_enabled><class id="Webhook_Uri_text" style="color:black;">URI</class></label>
+			</td>
+			<td>
+				<input required type="text" id="Webhook_Uri_value1">
+			</td>
+			<td>$TOOLTIP_Webhook_Uri</td>
+		</tr>
+
+        <tr class="WebhookItem">
+			<td class="indent1">
+				<input type="checkbox" id="Webhook_ApiKey_enabled" value="1"  onclick = 'InvertEnableItem("Webhook", "ApiKey")' unchecked >
+				<label for=Webhook_ApiKey_enabled><class id="Webhook_ApiKey_text" style="color:black;">ApiKey</class></label>
+			</td>
+			<td>
+				<input required type="text" id="Webhook_ApiKey_value1">
+			</td>
+			<td>$TOOLTIP_Webhook_ApiKey</td>
+		</tr>
+
 
 		<!------------- GPIO ------------------>
 		<tr style="border-bottom: 2px solid lightgray;">
@@ -2169,6 +2200,9 @@ function UpdateInput() {
     document.getElementById("Category_InfluxDBv2_enabled").checked = category["InfluxDBv2"]["enabled"];
     setVisible("InfluxDBv2Item", category["InfluxDBv2"]["enabled"]);
 
+	document.getElementById("Category_Webhook_enabled").checked = category["Webhook"]["enabled"];
+    setVisible("WebhookItem", category["Webhook"]["enabled"]);
+
     WriteParameter(param, category, "TakeImage", "RawImagesLocation", true);
     WriteParameter(param, category, "TakeImage", "RawImagesRetention", true);
 
@@ -2247,6 +2281,9 @@ function UpdateInput() {
     WriteParameter(param, category, "InfluxDBv2", "Token", true);	
     // WriteParameter(param, category, "InfluxDBv2", "Field", true);
 
+	WriteParameter(param, category, "Webhook", "Uri", true);	
+    WriteParameter(param, category, "Webhook", "ApiKey", true);
+
     WriteParameter(param, category, "GPIO", "IO0", true);
     WriteParameter(param, category, "GPIO", "IO1", true);
     WriteParameter(param, category, "GPIO", "IO3", true);
@@ -2407,6 +2444,9 @@ function ReadParameterAll() {
     ReadParameter(param, "InfluxDBv2", "Token", true);
     // ReadParameter(param, "InfluxDB", "Field", true);	
 
+	ReadParameter(param, "Webhook", "Uri", true);	
+    ReadParameter(param, "Webhook", "ApiKey", true);
+
     ReadParameter(param, "GPIO", "IO0", true);
     ReadParameter(param, "GPIO", "IO1", true);
     ReadParameter(param, "GPIO", "IO3", true);
@@ -2455,6 +2495,7 @@ function UpdateAfterCategoryCheck() {
     category["InfluxDB"]["enabled"] = document.getElementById("Category_InfluxDB_enabled").checked;
     category["InfluxDBv2"]["enabled"] = document.getElementById("Category_InfluxDBv2_enabled").checked;
     category["GPIO"]["enabled"] = document.getElementById("Category_GPIO_enabled").checked;
+	category["Webhook"]["enabled"] = document.getElementById("Category_Webhook_enabled").checked;
 
     UpdateInput();
     var sel = document.getElementById("Numbers_value1");

sd-card/html/readconfigparam.js

@@ -228,6 +228,14 @@ function ParseConfig() {
     ParamAddValue(param, catname, "Measurement", 1, true);
     ParamAddValue(param, catname, "Field", 1, true);
 
+    var catname = "Webhook";
+    category[catname] = new Object();
+    category[catname]["enabled"] = false;
+    category[catname]["found"] = false;
+    param[catname] = new Object();
+    ParamAddValue(param, catname, "Uri");
+    ParamAddValue(param, catname, "ApiKey");
+
     var catname = "GPIO";
     category[catname] = new Object();
     category[catname]["enabled"] = false;