Improve NTP handling (#1676)

* fix special case where number is named "default" (keep all topics in top level instead of in a sub-group)

* re-implemented SNTP usage, added way to disable NTP client, added timezone table

* minor fixes

Co-authored-by: CaCO3 <caco@ruinelli.ch>

code/components/jomjol_fileserver_ota/server_ota.cpp

@@ -45,6 +45,7 @@ static char ota_write_data[SERVER_OTA_SCRATCH_BUFSIZE + 1] = { 0 };
 static const char *TAG = "OTA";
 
 esp_err_t handler_reboot(httpd_req_t *req);
+static bool ota_update_task(std::string fn);
 
 std::string _file_name_update;
 

code/components/jomjol_fileserver_ota/server_ota.h

@@ -15,6 +15,5 @@ void CheckOTAUpdate();
 void doReboot();
 void hard_restart();
 void CheckUpdate();
-static bool ota_update_task(std::string fn);
 
 #endif //SERVEROTA_H

code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp

@@ -727,7 +727,7 @@ bool ClassFlowCNNGeneral::doNeuralNetwork(string time)
                             GENERAL[n]->ROI[roi]->isReject = true;
                             result = -1;
                             _result_save_file+= 100;     // In case fit is not sufficient, the result should still be saved with "-10x.y".
-                            string zw = "Value Rejected due to Threshold (Fit: " + to_string(_fit) + "Threshold: " + to_string(CNNGoodThreshold) + ")";
+                            string zw = "Value Rejected due to Threshold (Fit: " + to_string(_fit) + ", Threshold: " + to_string(CNNGoodThreshold) + ")";
                             LogFile.WriteToFile(ESP_LOG_WARN, TAG, zw);
                         }
                         else

code/components/jomjol_flowcontroll/ClassFlowControll.cpp

@@ -292,7 +292,7 @@ void ClassFlowControll::doFlowMakeImageOnly(string time){
     for (int i = 0; i < FlowControll.size(); ++i)
     {
         if (FlowControll[i]->name() == "ClassFlowMakeImage") {
-            zw_time = gettimestring("%H:%M:%S");
+            zw_time = getCurrentTimeString("%H:%M:%S");
             std::string flowStatus = TranslateAktstatus(FlowControll[i]->name());
             aktstatus = flowStatus + " (" + zw_time + ")";
 #ifdef ENABLE_MQTT
@@ -315,16 +315,19 @@ bool ClassFlowControll::doFlow(string time)
 #endif
 
     /* Check if we have a valid date/time and if not restart the NTP client */
-    if (! getTimeIsSet()) {
+   /* if (! getTimeIsSet()) {
         LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Time not set, restarting NTP Client!");
         restartNtpClient();
-    }
+    }*/
+
+    //checkNtpStatus(0);
 
     for (int i = 0; i < FlowControll.size(); ++i)
     {
-        zw_time = gettimestring("%H:%M:%S");
+        zw_time = getCurrentTimeString("%H:%M:%S");
         std::string flowStatus = TranslateAktstatus(FlowControll[i]->name());
         aktstatus = flowStatus + " (" + zw_time + ")";
+        //LogFile.WriteToFile(ESP_LOG_INFO, TAG, aktstatus);
 #ifdef ENABLE_MQTT
         MQTTPublish(mqttServer_getMainTopic() + "/" + "status", flowStatus, false);
 #endif //ENABLE_MQTT
@@ -355,9 +358,10 @@ bool ClassFlowControll::doFlow(string time)
 #endif
 
     }
-    zw_time = gettimestring("%H:%M:%S");
+    zw_time = getCurrentTimeString("%H:%M:%S");
     std::string flowStatus = "Flow finished";
     aktstatus = flowStatus + " (" + zw_time + ")";
+    //LogFile.WriteToFile(ESP_LOG_INFO, TAG, aktstatus);
 #ifdef ENABLE_MQTT
     MQTTPublish(mqttServer_getMainTopic() + "/" + "status", flowStatus, false);
 #endif //ENABLE_MQTT
@@ -541,17 +545,7 @@ bool ClassFlowControll::ReadParameter(FILE* pfile, string& aktparamgraph)
             LogFile.SetLogFileRetention(std::stoi(splitted[1]));
         }
 
-        if ((toUpper(splitted[0]) == "TIMEZONE") && (splitted.size() > 1))
-        {
-            string zw = "Set TimeZone: " + splitted[1];
-            setTimeZone(splitted[1]);
-        }      
-
-        if ((toUpper(splitted[0]) == "TIMESERVER") && (splitted.size() > 1))
-        {
-            string zw = "Set TimeZone: " + splitted[1];
-            reset_servername(splitted[1]);
-        }  
+        /* TimeServer and TimeZone got already read from the config, see setupTime () */
 
         if ((toUpper(splitted[0]) == "RSSITHREASHOLD") && (splitted.size() > 1))
         {

code/components/jomjol_helper/Helper.cpp

@@ -785,11 +785,17 @@ bool isSetSystemStatusFlag(SystemStatusFlag_t flag) {
 	}
 }
 
+
+time_t getUpTime(void) {
+    return (uint32_t)(esp_timer_get_time()/1000/1000); // in seconds
+}
+
+
 string getResetReason(void) {
 	std::string reasonText;
 
 	switch(esp_reset_reason()) {
-		case ESP_RST_POWERON: reasonText = "Power-on event"; break;    //!< Reset due to power-on event
+		case ESP_RST_POWERON: reasonText = "Power-on event (or reset button)"; break;    //!< Reset due to power-on event
 		case ESP_RST_EXT: reasonText = "External pin"; break;        //!< Reset by external pin (not applicable for ESP32)
 		case ESP_RST_SW: reasonText = "Via esp_restart"; break;         //!< Software reset via esp_restart
 		case ESP_RST_PANIC: reasonText = "Exception/panic"; break;      //!< Software reset due to exception/panic
@@ -814,7 +820,7 @@ std::string getFormatedUptime(bool compact) {
 	char buf[20];
 	#pragma GCC diagnostic ignored "-Wformat-truncation"
 
-    int uptime = (uint32_t)(esp_timer_get_time()/1000/1000); // in seconds
+    int uptime = getUpTime(); // in seconds
 
     int days = int(floor(uptime / (3600*24)));
     int hours = int(floor((uptime - days * 3600*24) / (3600)));

code/components/jomjol_helper/Helper.h

@@ -85,6 +85,7 @@ void clearSystemStatusFlag(SystemStatusFlag_t flag);
 int getSystemStatus(void);
 bool isSetSystemStatusFlag(SystemStatusFlag_t flag);
 
+time_t getUpTime(void);
 string getResetReason(void);
 std::string getFormatedUptime(bool compact);
 

code/components/jomjol_mqtt/server_mqtt.cpp

@@ -91,7 +91,13 @@ void sendHomeAssistantDiscoveryTopic(std::string group, std::string field,
         }
     }
     else {
+        if (field == "problem") { // Special binary sensor which is based on error topic
+            payload += "\"state_topic\": \"~/error\"," + nl;
+            payload += "\"value_template\": \"{{ 'OFF' if 'no error' in value else 'ON'}}\"," + nl;
+        }
+        else {
             payload += "\"state_topic\": \"~/" + field + "\"," + nl;
+        }
     }
 
     if (unit != "") {
@@ -149,7 +155,11 @@ void MQTThomeassistantDiscovery() {
 
 
     for (int i = 0; i < (*NUMBERS).size(); ++i) {
-         std::string group = (*NUMBERS)[i]->name;
+        std::string group = (*NUMBERS)[i]->name;
+        if (group == "default") {
+            group = "";
+        }
+
     //                                  Group | Field                 | User Friendly Name                | Icon                   | Unit     | Device Class | State Class       | Entity Category
         sendHomeAssistantDiscoveryTopic(group,   "value",              "Value",                            "gauge",                 valueUnit, meterType,     "total_increasing", "");
         sendHomeAssistantDiscoveryTopic(group,   "raw",                "Raw Value",                        "raw",                   valueUnit, "",            "total_increasing", "diagnostic");

code/components/jomjol_tfliteclass/server_tflite.cpp

@@ -101,7 +101,7 @@ void doInit(void)
 
 bool doflow(void)
 {   
-    std::string zw_time = gettimestring(LOGFILE_TIME_FORMAT);
+    std::string zw_time = getCurrentTimeString(LOGFILE_TIME_FORMAT);
     ESP_LOGD(TAG, "doflow - start %s", zw_time.c_str());
     flowisrunning = true;
     tfliteflow.doFlow(zw_time);
@@ -787,7 +787,7 @@ void task_autodoFlow(void *pvParameter)
     auto_isrunning = tfliteflow.isAutoStart(auto_intervall);
     if (isSetupModusActive()) {
         auto_isrunning = false;
-        std::string zw_time = gettimestring(LOGFILE_TIME_FORMAT);
+        std::string zw_time = getCurrentTimeString(LOGFILE_TIME_FORMAT);
         tfliteflow.doFlowMakeImageOnly(zw_time);
 
     }

code/components/jomjol_time_sntp/CMakeLists.txt

@@ -2,6 +2,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*)
 
 idf_component_register(SRCS ${app_sources}
                     INCLUDE_DIRS "."
-                    REQUIRES tflite-lib jomjol_logfile)
+                    REQUIRES tflite-lib jomjol_logfile jomjol_configfile)
 
 

code/components/jomjol_time_sntp/time_sntp.cpp

@@ -16,18 +16,20 @@
 
 #include "ClassLogFile.h"
 
+#include "configFile.h"
+#include "Helper.h"
+
+
 static const char *TAG = "SNTP";
 
-time_t bootTime;
+static std::string timeZone = "";
+static std::string timeServer = "undefined";
+static bool useNtp = true;
 
-static bool obtain_time(void);
-static void initialize_sntp(void);
-static void logNtpStatus(sntp_sync_status_t status);
+std::string getNtpStatusText(sntp_sync_status_t status);
+static void setTimeZone(std::string _tzstring);
+static std::string getServerName(void);
 
-void time_sync_notification_cb(struct timeval *tv)
-{
-    ESP_LOGI(TAG, "Notification of a time synchronization event");
-}
 
 std::string ConvertTimeToString(time_t _time, const char * frm)
 {
@@ -40,7 +42,8 @@ std::string ConvertTimeToString(time_t _time, const char * frm)
     return result;
 }
 
-std::string gettimestring(const char * frm)
+
+std::string getCurrentTimeString(const char * frm)
 {
     time_t now;
     struct tm timeinfo;
@@ -53,40 +56,14 @@ std::string gettimestring(const char * frm)
     return result;
 }
 
-bool setup_time()
+
+void time_sync_notification_cb(struct timeval *tv)
 {
-    time_t now;
-    struct tm timeinfo;
-    time(&now);
-    localtime_r(&now, &timeinfo);
-    char strftime_buf[64];
-    bool success = true;
-
-    // Is time set? If not, tm_year will be (1970 - 1900).
-    if (!getTimeIsSet()) {
-        initialize_sntp();
-        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Time is not set yet. Getting time over NTP server " + std::string(sntp_getservername(0)));
-        if (!obtain_time()) {
-            success = false;
-        }
-
-        // update 'now' variable with current time
-        time(&now);
-
-        setTimeZone("CET-1CEST,M3.5.0,M10.5.0/3");
-
-        localtime_r(&now, &timeinfo);
-        strftime(strftime_buf, sizeof(strftime_buf), "%Y-%m-%d_%H:%M:%S", &timeinfo);
-        ESP_LOGI(TAG, "The current date/time in Berlin is: %s", strftime_buf);
-    }
-    else {
-        localtime_r(&now, &timeinfo);
-        strftime(strftime_buf, sizeof(strftime_buf), "%Y-%m-%d_%H:%M:%S", &timeinfo);
-        ESP_LOGI(TAG, "Time is already set (%s)", strftime_buf);
-    }
-    return success;
+    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Time is now successfully synced with NTP Server " +
+            getServerName() + ": " + getCurrentTimeString("%Y-%m-%d %H:%M:%S"));
 }
 
+
 void setTimeZone(std::string _tzstring)
 {
     setenv("TZ", _tzstring.c_str(), 1);
@@ -95,100 +72,25 @@ void setTimeZone(std::string _tzstring)
     LogFile.WriteToFile(ESP_LOG_INFO, TAG, _tzstring);
 }
 
-static bool obtain_time(void)
-{
-    time_t now = 0;
-    struct tm timeinfo = {};
-    int retry = 0;
-    const int retry_count = 10;
-    bool success = true;
 
-    time(&now);
-    localtime_r(&now, &timeinfo);
-
-    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Waiting until we get a time from the NTP server " + std::string(sntp_getservername(0)));
-    while (true) {
-        retry++;
-
-        if (retry == retry_count) {
-            ESP_LOGW(TAG, "NTP time fetching seems to take longer, will check again on next round!"); // The NTP client will automatically retry periodically!
-            success = false;
-            break;
-        }
-
-        sntp_sync_status_t status = sntp_get_sync_status();
-        logNtpStatus(status);
-        if (status == SNTP_SYNC_STATUS_COMPLETED) {
-            LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Time is synced with NTP Server " + std::string(sntp_getservername(0)));
-            break;
-        }
-
-        vTaskDelay(2000 / portTICK_PERIOD_MS);
-    }
-    
-    time(&now);
-    localtime_r(&now, &timeinfo);
-    return success;
-}
-
-
-void logNtpStatus(sntp_sync_status_t status) {
+std::string getNtpStatusText(sntp_sync_status_t status) {
     if (status == SNTP_SYNC_STATUS_COMPLETED) {
-        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Status OK");
+        return "Synchronized";
     }
     else if (status == SNTP_SYNC_STATUS_IN_PROGRESS) {
-        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Status: In Progress");
+        return "In Progress";
     }
     else { // SNTP_SYNC_STATUS_RESET
-        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Status: Reset");
+        return "Reset";
     }
 }
 
 
-void reset_servername(std::string _servername)
-{
-    sntp_stop();
-    sntp_setoperatingmode(SNTP_OPMODE_POLL);
-    sntp_setservername(0, _servername.c_str());
-    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Set SNTP-Server to " + std::string(sntp_getservername(0)));
-    sntp_init();
-    obtain_time();
-    std::string zw = gettimestring("%Y%m%d-%H%M%S");
-    ESP_LOGD(TAG, "Time ist %s", zw.c_str());
-}
-
-static void initialize_sntp(void)
-{
-    ESP_LOGI(TAG, "Initializing SNTP");
-    sntp_setoperatingmode(SNTP_OPMODE_POLL);
-    sntp_setservername(0, "pool.ntp.org");
-//    sntp_set_time_sync_notification_cb(time_sync_notification_cb);
-    sntp_init();
-}
-
-void setBootTime() 
-{
-    time(&bootTime);
-}
-
-time_t getUpTime() 
-{
-    time_t now;
-    time(&now);
-
-    return now - bootTime;
-}
-
 bool getTimeIsSet(void) {
     time_t now;
     struct tm timeinfo;
     time(&now);
     localtime_r(&now, &timeinfo);
-    char strftime_buf[64];
-
-    localtime_r(&now, &timeinfo);
-    strftime(strftime_buf, sizeof(strftime_buf), "%Y-%m-%d_%H:%M:%S", &timeinfo);
-    ESP_LOGD(TAG, "The current date/time in Berlin is: %s", strftime_buf);
 
     // Is time set? If not, tm_year will be (1970 - 1900).
     if ((timeinfo.tm_year < (2022 - 1900))) {
@@ -199,7 +101,127 @@ bool getTimeIsSet(void) {
     }
 }
 
-void restartNtpClient(void) {
-    sntp_restart();
-    obtain_time();
+/*void restartNtpClient(void) {
+//    sntp_restart();
+ //   obtain_time();
+}*/
+
+
+bool getUseNtp(void) {
+    return useNtp;
+}
+
+
+std::string getServerName(void) {
+    char buf[100];
+
+    if (sntp_getservername(0)){
+        snprintf(buf, sizeof(buf), "%s", sntp_getservername(0));
+        return std::string(buf);
+    }
+    else { // we have either IPv4 or IPv6 address
+        ip_addr_t const *ip = sntp_getserver(0);
+        if (ipaddr_ntoa_r(ip, buf, sizeof(buf)) != NULL) {
+            return std::string(buf);
+        }
+    }
+    return "";
+}
+
+
+/**
+ * Load the TimeZone and TimeServer from the config file and initialize the NTP client
+ */
+bool setupTime() {
+    time_t now;
+    struct tm timeinfo;
+    char strftime_buf[64];
+
+    ConfigFile configFile = ConfigFile(CONFIG_FILE); 
+
+    std::vector<std::string> splitted;
+    std::string line = "";
+    bool disabledLine = false;
+    bool eof = false;
+
+    /* Load config from config file */
+    while ((!configFile.GetNextParagraph(line, disabledLine, eof) || 
+            (line.compare("[System]") != 0)) && !eof) {}
+    if (eof) {
+        return false;
+    }
+
+    if (disabledLine) {
+        return false;
+    }
+
+    while (configFile.getNextLine(&line, disabledLine, eof) && 
+            !configFile.isNewParagraph(line)) {
+        splitted = ZerlegeZeile(line);
+
+        if (toUpper(splitted[0]) == "TIMEZONE") {
+            timeZone = splitted[1];
+        }
+
+        if (toUpper(splitted[0]) == "TIMESERVER") {
+            if (splitted.size() <= 1) { // Key has no value => we use this to show it as disabled
+                timeServer = "";
+            }
+            else {
+                timeServer = splitted[1];
+            }
+        }
+    }
+
+
+    /* Setup NTP Server and Timezone */
+    if (timeServer == "undefined") {
+        timeServer = "pool.ntp.org";
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "TimeServer not defined, using default: " + timeServer);
+    }
+    else if (timeServer == "") {
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "TimeServer config empty, disabling NTP");
+        useNtp = false;
+    }
+    else {
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "TimeServer: " + timeServer);
+    }
+    
+    if (timeZone == "") {
+        timeZone = "CET-1CEST,M3.5.0,M10.5.0/3";
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "TimeZone not set, using default: " + timeZone);
+    }
+
+
+    if (useNtp) {
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Configuring NTP Client...");        
+        sntp_setoperatingmode(SNTP_OPMODE_POLL);
+        sntp_setservername(0, timeServer.c_str());
+        sntp_init();
+
+        sntp_set_time_sync_notification_cb(time_sync_notification_cb);
+
+        setTimeZone(timeZone);
+    }
+
+
+    /* The RTC keeps the time after a restart (Except on Power On or Pin Reset) 
+     * There should only be a minor correction through NTP */
+
+    // Get current time from RTC
+    time(&now);
+    localtime_r(&now, &timeinfo);
+    strftime(strftime_buf, sizeof(strftime_buf), "%Y-%m-%d %H:%M:%S", &timeinfo);
+
+    if (getTimeIsSet()) {
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Time is already set: " + std::string(strftime_buf));
+    }
+    else {
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "The local time is unknown, starting with " + std::string(strftime_buf));
+        if (useNtp) {
+            LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Once the NTP server provides a time, we will switch to that one");
+        }
+    }
+
+    return true;
 }

code/components/jomjol_time_sntp/time_sntp.h

@@ -15,20 +15,15 @@
 #include "esp_log.h"
 #include "esp_attr.h"
 #include "esp_sleep.h"
-// #include "nvs_flash.h"
 #include "esp_sntp.h"
 
-bool setup_time(void);
-
-std::string gettimestring(const char * frm);
+std::string getCurrentTimeString(const char * frm);
 std::string ConvertTimeToString(time_t _time, const char * frm);
 
-void setTimeZone(std::string _tzstring);
-void reset_servername(std::string _servername);
 
-void setBootTime();
-time_t getUpTime();
 bool getTimeIsSet(void);
-void restartNtpClient(void);
+
+bool getUseNtp(void);
+bool setupTime();
 
 #endif //TIMESNTP_H