Implement InfluxDB v2 (#2004)

* Implement InfluxDBv2 Innitial

* Update incl. UI

* Correct UI

* Update UI - Indiv. Param.

* Update edit_config_param.html

* Correct Timeshift

* Update Fieldname

code/components/jomjol_flowcontroll/ClassFlowControll.cpp

@@ -60,6 +60,9 @@ std::string ClassFlowControll::doSingleStep(std::string _stepname, std::string _
     if ((_stepname.compare("[InfluxDB]") == 0) || (_stepname.compare(";[InfluxDB]") == 0)){
         _classname = "ClassFlowInfluxDB";
     }
+    if ((_stepname.compare("[InfluxDBv2]") == 0) || (_stepname.compare(";[InfluxDBv2]") == 0)){
+        _classname = "ClassFlowInfluxDBv2";
+    }
     #endif //ENABLE_INFLUXDB
 
     for (int i = 0; i < FlowControll.size(); ++i)
@@ -90,6 +93,8 @@ std::string ClassFlowControll::TranslateAktstatus(std::string _input)
     #ifdef ENABLE_INFLUXDB
         if (_input.compare("ClassFlowInfluxDB") == 0)
             return ("Sending InfluxDB");
+        if (_input.compare("ClassFlowInfluxDBv2") == 0)
+            return ("Sending InfluxDBv2");
     #endif //ENABLE_INFLUXDB
     if (_input.compare("ClassFlowPostProcessing") == 0)
         return ("Post-Processing");
@@ -233,6 +238,8 @@ ClassFlow* ClassFlowControll::CreateClassFlow(std::string _type)
     #ifdef ENABLE_INFLUXDB
     if (toUpper(_type).compare("[INFLUXDB]") == 0)
         cfc = new ClassFlowInfluxDB(&FlowControll);
+    if (toUpper(_type).compare("[INFLUXDBV2]") == 0)
+        cfc = new ClassFlowInfluxDBv2(&FlowControll);
     #endif //ENABLE_INFLUXDB  
     if (toUpper(_type).compare("[WRITELIST]") == 0)
         cfc = new ClassFlowWriteList(&FlowControll);
@@ -290,6 +297,7 @@ void ClassFlowControll::InitFlow(std::string config)
     while ((line.size() > 0) && !(feof(pFile)))
     {
         cfc = CreateClassFlow(line);
+//        printf("Name: %s\n", cfc->name().c_str());
         if (cfc)
         {
             ESP_LOGD(TAG, "Start ReadParameter (%s)", line.c_str());

code/components/jomjol_flowcontroll/ClassFlowControll.h

@@ -15,6 +15,7 @@
 #endif //ENABLE_MQTT
 #ifdef ENABLE_INFLUXDB
 	#include "ClassFlowInfluxDB.h"
+	#include "ClassFlowInfluxDBv2.h"
 #endif //ENABLE_INFLUXDB
 #include "ClassFlowCNNGeneral.h"
 #include "ClassFlowWriteList.h"

code/components/jomjol_flowcontroll/ClassFlowDefineTypes.h

@@ -50,6 +50,7 @@ struct NumberPost {
     int DecimalShiftInitial;
     float AnalogDigitalTransitionStart; // When is the digit > x.1, i.e. when does it start to tilt?
     int Nachkomma;
+    string Fieldname; // Fieldname in InfluxDB2  
 
     bool isExtendedResolution;
 

code/components/jomjol_flowcontroll/ClassFlowInfluxDBv2.cpp

@@ -0,0 +1,219 @@
+#ifdef ENABLE_INFLUXDB
+#include <sstream>
+#include "ClassFlowInfluxDBv2.h"
+#include "Helper.h"
+#include "connect_wlan.h"
+
+#include "time_sntp.h"
+#include "interface_influxdb.h"
+
+#include "ClassFlowPostProcessing.h"
+#include "esp_log.h"
+#include "../../include/defines.h"
+
+#include "ClassLogFile.h"
+
+#include <time.h>
+
+static const char* TAG = "class_flow_influxDbv2";
+
+void ClassFlowInfluxDBv2::SetInitialParameter(void)
+{
+    uri = "";
+    database = "";
+    measurement = "";
+    dborg = "";  
+    dbtoken = "";  
+//    dbfield = "";
+
+    OldValue = "";
+    flowpostprocessing = NULL;  
+    previousElement = NULL;
+    ListFlowControll = NULL; 
+    disabled = false;
+    InfluxDBenable = false;
+}       
+
+ClassFlowInfluxDBv2::ClassFlowInfluxDBv2()
+{
+    SetInitialParameter();
+}
+
+ClassFlowInfluxDBv2::ClassFlowInfluxDBv2(std::vector<ClassFlow*>* lfc)
+{
+    SetInitialParameter();
+
+    ListFlowControll = lfc;
+    for (int i = 0; i < ListFlowControll->size(); ++i)
+    {
+        if (((*ListFlowControll)[i])->name().compare("ClassFlowPostProcessing") == 0)
+        {
+            flowpostprocessing = (ClassFlowPostProcessing*) (*ListFlowControll)[i];
+        }
+    }
+}
+
+ClassFlowInfluxDBv2::ClassFlowInfluxDBv2(std::vector<ClassFlow*>* lfc, ClassFlow *_prev)
+{
+    SetInitialParameter();
+
+    previousElement = _prev;
+    ListFlowControll = lfc;
+
+    for (int i = 0; i < ListFlowControll->size(); ++i)
+    {
+        if (((*ListFlowControll)[i])->name().compare("ClassFlowPostProcessing") == 0)
+        {
+            flowpostprocessing = (ClassFlowPostProcessing*) (*ListFlowControll)[i];
+        }
+    }
+}
+
+
+bool ClassFlowInfluxDBv2::ReadParameter(FILE* pfile, string& aktparamgraph)
+{
+    std::vector<string> splitted;
+
+    aktparamgraph = trim(aktparamgraph);
+    printf("akt param: %s\n", aktparamgraph.c_str());
+
+    if (aktparamgraph.size() == 0)
+        if (!this->GetNextParagraph(pfile, aktparamgraph))
+            return false;
+
+    if (toUpper(aktparamgraph).compare("[INFLUXDBV2]") != 0) 
+        return false;
+
+    while (this->getNextLine(pfile, &aktparamgraph) && !this->isNewParagraph(aktparamgraph))
+    {
+//        ESP_LOGD(TAG, "while loop reading line: %s", aktparamgraph.c_str());
+        splitted = ZerlegeZeile(aktparamgraph);
+        std::string _param = GetParameterName(splitted[0]);
+
+        if ((toUpper(_param) == "ORG") && (splitted.size() > 1))
+        {
+            this->dborg = splitted[1];
+        }  
+        if ((toUpper(_param) == "TOKEN") && (splitted.size() > 1))
+        {
+            this->dbtoken = splitted[1];
+        }               
+        if ((toUpper(_param) == "URI") && (splitted.size() > 1))
+        {
+            this->uri = splitted[1];
+        }
+        if (((toUpper(_param) == "MEASUREMENT")) && (splitted.size() > 1))
+        {
+            this->measurement = splitted[1];
+        }
+        if (((toUpper(_param) == "FIELDNAME")) && (splitted.size() > 1))
+        {
+            handleFieldname(splitted[0], splitted[1]);
+        }
+        if (((toUpper(splitted[0]) == "DATABASE")) && (splitted.size() > 1))
+        {
+            this->database = splitted[1];
+        }
+    }
+
+    printf("uri:         %s\n", uri.c_str());
+    printf("measurement: %s\n", measurement.c_str());
+    printf("org:         %s\n", dborg.c_str());
+    printf("token:       %s\n", dbtoken.c_str());
+
+    if ((uri.length() > 0) && (database.length() > 0) && (measurement.length() > 0) && (dbtoken.length() > 0) && (dborg.length() > 0)) 
+    { 
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Init InfluxDB with uri: " + uri + ", measurement: " + measurement + ", org: " + dborg + ", token: *****");
+//        printf("vor V2 Init\n");
+        InfluxDB_V2_Init(uri, database, measurement, dborg, dbtoken); 
+//        printf("nach V2 Init\n");
+        InfluxDBenable = true;
+    } else {
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "InfluxDBv2 (Verion2 !!!) init skipped as we are missing some parameters");
+    }
+   
+    return true;
+}
+
+
+string ClassFlowInfluxDBv2::GetInfluxDBMeasurement()
+{
+    return measurement;
+}
+
+void ClassFlowInfluxDBv2::handleFieldname(string _decsep, string _value)
+{
+    string _digit, _decpos;
+    int _pospunkt = _decsep.find_first_of(".");
+//    ESP_LOGD(TAG, "Name: %s, Pospunkt: %d", _decsep.c_str(), _pospunkt);
+    if (_pospunkt > -1)
+        _digit = _decsep.substr(0, _pospunkt);
+    else
+        _digit = "default";
+    for (int j = 0; j < flowpostprocessing->NUMBERS.size(); ++j)
+    {
+        if (_digit == "default")                        //  Set to default first (if nothing else is set)
+        {
+            flowpostprocessing->NUMBERS[j]->Fieldname = _value;
+        }
+        if (flowpostprocessing->NUMBERS[j]->name == _digit)
+        {
+            flowpostprocessing->NUMBERS[j]->Fieldname = _value;
+        }
+    }
+}
+
+
+
+bool ClassFlowInfluxDBv2::doFlow(string zwtime)
+{
+    if (!InfluxDBenable)
+        return true;
+
+    std::string result;
+    std::string resulterror = "";
+    std::string resultraw = "";
+    std::string resultrate = "";
+    std::string resulttimestamp = "";
+    string zw = "";
+    string namenumber = "";
+
+    if (flowpostprocessing)
+    {
+        std::vector<NumberPost*>* NUMBERS = flowpostprocessing->GetNumbers();
+
+        for (int i = 0; i < (*NUMBERS).size(); ++i)
+        {
+            result =  (*NUMBERS)[i]->ReturnValue;
+            resultraw =  (*NUMBERS)[i]->ReturnRawValue;
+            resulterror = (*NUMBERS)[i]->ErrorMessageText;
+            resultrate = (*NUMBERS)[i]->ReturnRateValue;
+            resulttimestamp = (*NUMBERS)[i]->timeStamp;
+
+            if ((*NUMBERS)[i]->Fieldname.length() > 0)
+            {
+                namenumber = (*NUMBERS)[i]->Fieldname;
+            }
+            else
+            {
+                namenumber = (*NUMBERS)[i]->name;
+                if (namenumber == "default")
+                    namenumber = "value";
+                else
+                    namenumber = namenumber + "/value";
+            }
+            
+            printf("vor sende Influx_DB_V2 - namenumber. %s, result: %s, timestampt: %s", namenumber.c_str(), result.c_str(), resulttimestamp.c_str());
+
+            if (result.length() > 0)   
+                InfluxDB_V2_Publish(namenumber, result, resulttimestamp);
+//                InfluxDB_V2_Publish(namenumber, result, resulttimestamp);
+        }
+    }
+   
+    OldValue = result;
+    
+    return true;
+}
+
+#endif //ENABLE_INFLUXDB

code/components/jomjol_flowcontroll/ClassFlowInfluxDBv2.h

@@ -0,0 +1,41 @@
+#ifdef ENABLE_INFLUXDB
+
+#pragma once
+
+#ifndef CLASSFINFLUXDBv2_H
+#define CLASSFINFLUXDBv2_H
+
+#include "ClassFlow.h"
+
+#include "ClassFlowPostProcessing.h"
+
+#include <string>
+
+class ClassFlowInfluxDBv2 :
+    public ClassFlow
+{
+protected:
+    std::string uri, database, measurement;
+    std::string dborg, dbtoken, dbfield;
+    std::string OldValue;
+	ClassFlowPostProcessing* flowpostprocessing;  
+    bool InfluxDBenable;
+
+    void SetInitialParameter(void);     
+
+    void handleFieldname(string _decsep, string _value);   
+
+public:
+    ClassFlowInfluxDBv2();
+    ClassFlowInfluxDBv2(std::vector<ClassFlow*>* lfc);
+    ClassFlowInfluxDBv2(std::vector<ClassFlow*>* lfc, ClassFlow *_prev);
+
+    string GetInfluxDBMeasurement();
+
+    bool ReadParameter(FILE* pfile, string& aktparamgraph);
+    bool doFlow(string time);
+    string name(){return "ClassFlowInfluxDBv2";};
+};
+
+#endif //CLASSFINFLUXDBv2_H
+#endif //ENABLE_INFLUXDB

code/components/jomjol_flowcontroll/ClassFlowPostProcessing.h

@@ -15,7 +15,6 @@ class ClassFlowPostProcessing :
     public ClassFlow
 {
 protected:
-    std::vector<NumberPost*> NUMBERS;
     bool UpdatePreValueINI;
 
     int PreValueAgeStartup; 
@@ -54,6 +53,8 @@ protected:
 
 public:
     bool PreValueUse;
+    std::vector<NumberPost*> NUMBERS;
+
 
     ClassFlowPostProcessing(std::vector<ClassFlow*>* lfc, ClassFlowCNNGeneral *_analog, ClassFlowCNNGeneral *_digit);
     virtual ~ClassFlowPostProcessing(){};

code/components/jomjol_helper/Helper.cpp

@@ -576,9 +576,6 @@ std::vector<string> HelperZerlegeZeile(std::string input, std::string _delimiter
 std::vector<string> ZerlegeZeile(std::string input, std::string delimiter)
 {
 	std::vector<string> Output;
-
-	input = trim(input, delimiter);
-
 	/* The input can have multiple formats: 
 	 *  - key = value
      *  - key = value1 value2 value3 ...
@@ -593,12 +590,13 @@ std::vector<string> ZerlegeZeile(std::string input, std::string delimiter)
 	 * As a workaround and to not break any legacy usage, we enforce to only use the
 	 * equal sign, if the key is "password"
 	*/
-	if (input.find("password") != string::npos) { // Line contains a password, use the equal sign as the only delimiter and only split on first occurrence
+	if ((input.find("password") != string::npos) || (input.find("Token") != string::npos)) { // Line contains a password, use the equal sign as the only delimiter and only split on first occurrence
 		size_t pos = input.find("=");
 		Output.push_back(trim(input.substr(0, pos), ""));
 		Output.push_back(trim(input.substr(pos +1, string::npos), ""));
 	}
 	else { // Legacy Mode
+		input = trim(input, delimiter);							// sonst werden delimiter am Ende (z.B. == im Token) gelöscht)
 		size_t pos = findDelimiterPos(input, delimiter);
 		std::string token;
 		while (pos != std::string::npos) {

code/components/jomjol_influxdb/interface_influxdb.cpp

@@ -16,6 +16,101 @@ std::string _influxDBMeasurement;
 std::string _influxDBUser;
 std::string _influxDBPassword;
 
+std::string _influxDB_V2_URI;
+std::string _influxDB_V2_Database;
+std::string _influxDB_V2_Measurement;
+std::string _influxDB_V2_Token;
+std::string _influxDB_V2_Org;
+
+static esp_err_t http_event_handler(esp_http_client_event_t *evt);
+
+void InfluxDB_V2_Init(std::string _uri, std::string _database, std::string _measurement, std::string _org, std::string _token)
+{
+    _influxDB_V2_URI = _uri;
+    _influxDB_V2_Database = _database;
+    _influxDB_V2_Measurement = _measurement;
+    _influxDB_V2_Org = _org;
+    _influxDB_V2_Token = _token;
+}
+
+void InfluxDB_V2_Publish(std::string _key, std::string _content, std::string _timestamp) 
+{
+    char response_buffer[MAX_HTTP_OUTPUT_BUFFER] = {0};
+    esp_http_client_config_t http_config = {
+       .user_agent = "ESP32 Meter reader",
+       .method = HTTP_METHOD_POST,
+       .event_handler = http_event_handler,
+       .buffer_size = MAX_HTTP_OUTPUT_BUFFER,
+       .user_data = response_buffer
+    };
+
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "InfluxDB_V2_Publish - Key: " + _key + ", Content: " + _content + ", Timestamp: " + _timestamp);
+
+    // Format:     #define PREVALUE_TIME_FORMAT_OUTPUT "%Y-%m-%dT%H:%M:%S%z"
+
+    char nowTimestamp[21];
+    std::string payload;
+
+    if (_timestamp.length() > 0)
+    {
+        struct tm tm;
+        strptime(_timestamp.c_str(), PREVALUE_TIME_FORMAT_OUTPUT, &tm);
+        time_t t = mktime(&tm); // Time in Localtime (looks like timezone is not used by strptime)
+
+        struct tm * ptm;
+        ptm = gmtime ( &t );
+        time_t utc = mktime(ptm);
+        utc = 2*t - utc;        // Take care of timezone (looks difficult, but is easy: t = t + (t - utc), weil t-utc = timezone)
+
+        sprintf(nowTimestamp,"%ld000000000", (long) utc);           // UTC
+
+        payload = _influxDB_V2_Measurement + " " + _key + "=" + _content + " " + nowTimestamp;
+//        payload = _influxDB_V2_Measurement + " " + _key + "=774 " + nowTimestamp;
+    }
+    else
+    {
+        payload = _influxDB_V2_Measurement + " " + _key + "=" + _content;
+//        payload = _influxDB_V2_Measurement + " " + _key + "=774";
+    }
+
+    payload.shrink_to_fit();
+
+    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "sending line to influxdb:" + payload);
+
+    std::string apiURI = _influxDB_V2_URI + "/api/v2/write?org=" + _influxDB_V2_Org + "&bucket=" + _influxDB_V2_Database;
+    apiURI.shrink_to_fit();
+    http_config.url = apiURI.c_str();
+    ESP_LOGI(TAG, "http_config: %s", http_config.url); // Add mark on log to see when it restarted
+
+    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "API URI: " + apiURI);
+
+    esp_http_client_handle_t http_client = esp_http_client_init(&http_config);
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "client is initialized");
+
+    esp_http_client_set_header(http_client, "Content-Type", "text/plain");
+    std::string _zw = "Token " + _influxDB_V2_Token;
+    //    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Tokenheader: %s\n", _zw.c_str());
+    esp_http_client_set_header(http_client, "Authorization", _zw.c_str());
+
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "header is set");
+
+    ESP_ERROR_CHECK(esp_http_client_set_post_field(http_client, payload.c_str(), payload.length()));
+    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "post payload is set");
+
+    esp_err_t err = ESP_ERROR_CHECK_WITHOUT_ABORT(esp_http_client_perform(http_client));
+
+    if( err == ESP_OK ) {
+      LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP request was performed");
+      int status_code = esp_http_client_get_status_code(http_client);
+      LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP status code" + std::to_string(status_code));
+    } else {
+      LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "HTTP request failed");
+    }
+    esp_http_client_cleanup(http_client);
+}
+
+
+
 static esp_err_t http_event_handler(esp_http_client_event_t *evt)
 {
     switch(evt->event_id)
@@ -64,26 +159,30 @@ void InfluxDBPublish(std::string _key, std::string _content, std::string _timest
 
     LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "InfluxDBPublish - Key: " + _key + ", Content: " + _content + ", Timestamp: " + _timestamp);
 
-    // Format:     #define PREVALUE_TIME_FORMAT_OUTPUT "%Y-%m-%dT%H:%M:%S%z"
-    struct tm tm;
-    strptime(_timestamp.c_str(), PREVALUE_TIME_FORMAT_OUTPUT, &tm);
-
-    time_t t = mktime(&tm); // Time in Localtime (looks like timezone is not used by strptime)
-
-    struct tm * ptm;
-    ptm = gmtime ( &t );
-
-    time_t utc = mktime(ptm);
-    utc = 2*t - utc;
-
     char nowTimestamp[21];
+    std::string payload;
 
-    sprintf(nowTimestamp,"%ld000000000", (long) utc);           // UTC
+    if (_timestamp.length() > 0)
+    {
+        struct tm tm;
+        strptime(_timestamp.c_str(), PREVALUE_TIME_FORMAT_OUTPUT, &tm);
+        time_t t = mktime(&tm); // Time in Localtime (looks like timezone is not used by strptime)
 
+        struct tm * ptm;
+        ptm = gmtime ( &t );
+        time_t utc = mktime(ptm);
+        utc = 2*t - utc;        // Take care of timezone (looks difficult, but is easy: t = t + (t - utc), weil t-utc = timezone)
 
-//    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Test Time Conversion - t: " + std::to_string(t) + ", utc: " + std::to_string(utc) + ", now: " + std::to_string(now) + ", utc_local: " + std::to_string(utc_local));
+        sprintf(nowTimestamp,"%ld000000000", (long) utc);           // UTC
+
+        payload = _influxDB_V2_Measurement + " " + _key + "=" + _content + " " + nowTimestamp;
+//        payload = _influxDB_V2_Measurement + " " + _key + "=774 " + nowTimestamp;
+    }
+    else
+    {
+        payload = _influxDBMeasurement + " " + _key + "=" + _content;
+    }
 
-    std::string payload = _influxDBMeasurement + " " + _key + "=" + _content + " " + nowTimestamp;
     payload.shrink_to_fit();
 
     LogFile.WriteToFile(ESP_LOG_INFO, TAG, "sending line to influxdb:" + payload);

code/components/jomjol_influxdb/interface_influxdb.h

@@ -8,10 +8,17 @@
 #include <map>
 #include <functional>
 
+// Interface to InfluxDB v1.x
 void InfluxDBInit(std::string _influxDBURI, std::string _database, std::string _measurement, std::string _user, std::string _password);
-void InfluxDBdestroy();
-
 void InfluxDBPublish(std::string _key, std::string _content, std::string _timestamp);
 
+// Interface to InfluxDB v2.x
+void InfluxDB_V2_Init(std::string _uri, std::string _database, std::string _measurement, std::string _org, std::string _token);
+void InfluxDB_V2_Publish(std::string _key, std::string _content, std::string _timestamp);
+
+
+
+void InfluxDBdestroy();
+
 #endif //INTERFACE_INFLUXDB_H
 #endif //ENABLE_INFLUXDB