diff --git a/code/components/jomjol_flowcontroll/CMakeLists.txt b/code/components/jomjol_flowcontroll/CMakeLists.txt index 2b58ada9..4ad9f317 100644 --- a/code/components/jomjol_flowcontroll/CMakeLists.txt +++ b/code/components/jomjol_flowcontroll/CMakeLists.txt @@ -2,6 +2,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*) idf_component_register(SRCS ${app_sources} INCLUDE_DIRS "." - REQUIRES esp_timer esp_wifi jomjol_tfliteclass jomjol_helper jomjol_controlcamera jomjol_mqtt jomjol_influxdb jomjol_fileserver_ota jomjol_image_proc jomjol_wlan) + REQUIRES esp_timer esp_wifi jomjol_tfliteclass jomjol_helper jomjol_controlcamera jomjol_mqtt jomjol_influxdb jomjol_fileserver_ota jomjol_image_proc jomjol_wlan openmetrics) diff --git a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp index f54df869..b4b7bb18 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp @@ -927,3 +927,11 @@ string ClassFlowControll::getJSON() { return flowpostprocessing->GetJSON(); } + +/** + * @returns a vector of all current sequences + **/ +const std::vector &ClassFlowControll::getNumbers() +{ + return *flowpostprocessing->GetNumbers(); +} diff --git a/code/components/jomjol_flowcontroll/ClassFlowControll.h b/code/components/jomjol_flowcontroll/ClassFlowControll.h index e0d5f74e..3021338b 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowControll.h +++ b/code/components/jomjol_flowcontroll/ClassFlowControll.h @@ -52,6 +52,7 @@ public: string GetPrevalue(std::string _number = ""); bool ReadParameter(FILE* pfile, string& aktparamgraph); string getJSON(); + const std::vector &getNumbers(); string getNumbersName(); string TranslateAktstatus(std::string _input); diff --git a/code/components/jomjol_flowcontroll/MainFlowControl.cpp b/code/components/jomjol_flowcontroll/MainFlowControl.cpp index 63eadac1..b3c0a6a8 100644 --- a/code/components/jomjol_flowcontroll/MainFlowControl.cpp +++ b/code/components/jomjol_flowcontroll/MainFlowControl.cpp @@ -474,6 +474,71 @@ esp_err_t handler_json(httpd_req_t *req) return ESP_OK; } +/** + * Generates a http response containing the OpenMetrics (https://openmetrics.io/) text wire format + * according to https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#text-format. + * + * A MetricFamily with a Metric for each Sequence is provided. If no valid value is available, the metric is not provided. + * MetricPoints are provided without a timestamp. Additional metrics with some device information is also provided. + * + * The metric name prefix is 'ai_on_the_edge_device_'. + * + * example configuration for Prometheus (`prometheus.yml`): + * + * - job_name: watermeter + * static_configs: + * - targets: ['watermeter.fritz.box'] + * +*/ +esp_err_t handler_openmetrics(httpd_req_t *req) +{ +#ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("handler_openmetrics - Start"); +#endif + + ESP_LOGD(TAG, "handler_openmetrics uri: %s", req->uri); + + if (bTaskAutoFlowCreated) + { + httpd_resp_set_hdr(req, "Access-Control-Allow-Origin", "*"); + httpd_resp_set_type(req, "text/plain"); // application/openmetrics-text is not yet supported by prometheus so we use text/plain for now + + const string metricNamePrefix = "ai_on_the_edge_device"; + + // get current measurement (flow) + string response = createSequenceMetrics(metricNamePrefix, flowctrl.getNumbers()); + + // CPU Temperature + response += createMetric(metricNamePrefix + "_cpu_temperature_celsius", "current cpu temperature in celsius", "gauge", std::to_string((int)temperatureRead())); + + // WiFi signal strength + response += createMetric(metricNamePrefix + "_rssi_dbm", "current WiFi signal strength in dBm", "gauge", std::to_string(get_WIFI_RSSI())); + + // memory info + response += createMetric(metricNamePrefix + "_memory_heap_free_bytes", "available heap memory", "gauge", std::to_string(getESPHeapSize())); + + // device uptime + response += createMetric(metricNamePrefix + "_uptime_seconds", "device uptime in seconds", "gauge", std::to_string((long)getUpTime())); + + // data aquisition round + response += createMetric(metricNamePrefix + "_rounds_total", "data aquisition rounds since device startup", "counter", std::to_string(countRounds)); + + // the response always contains at least the metadata (HELP, TYPE) for the MetricFamily so no length check is needed + httpd_resp_send(req, response.c_str(), response.length()); + } + else + { + httpd_resp_send_err(req, HTTPD_403_FORBIDDEN, "Flow not (yet) started: REST API /metrics not yet available!"); + return ESP_ERR_NOT_FOUND; + } + +#ifdef DEBUG_DETAIL_ON + LogFile.WriteHeapInfo("handler_openmetrics - Done"); +#endif + + return ESP_OK; +} + esp_err_t handler_wasserzaehler(httpd_req_t *req) { #ifdef DEBUG_DETAIL_ON @@ -1650,4 +1715,12 @@ void register_server_main_flow_task_uri(httpd_handle_t server) camuri.handler = handler_stream; camuri.user_ctx = (void *)"stream"; httpd_register_uri_handler(server, &camuri); + + /** will handle metrics requests */ + camuri.uri = "/metrics"; + camuri.handler = handler_openmetrics; + camuri.user_ctx = (void *)"metrics"; + httpd_register_uri_handler(server, &camuri); + + /** when adding a new handler, make sure to increment the value for config.max_uri_handlers in `main/server_main.cpp` */ } diff --git a/code/components/jomjol_flowcontroll/MainFlowControl.h b/code/components/jomjol_flowcontroll/MainFlowControl.h index 8a2d1c41..2810837f 100644 --- a/code/components/jomjol_flowcontroll/MainFlowControl.h +++ b/code/components/jomjol_flowcontroll/MainFlowControl.h @@ -9,6 +9,7 @@ #include #include "CImageBasis.h" #include "ClassFlowControll.h" +#include "openmetrics.h" typedef struct { diff --git a/code/components/jomjol_helper/Helper.cpp b/code/components/jomjol_helper/Helper.cpp index c794646a..d6163155 100644 --- a/code/components/jomjol_helper/Helper.cpp +++ b/code/components/jomjol_helper/Helper.cpp @@ -1206,3 +1206,13 @@ bool isInString(std::string &s, std::string const &toFind) return true; } + +// from https://stackoverflow.com/a/14678800 +void replaceAll(std::string& s, const std::string& toReplace, const std::string& replaceWith) +{ + size_t pos = 0; + while ((pos = s.find(toReplace, pos)) != std::string::npos) { + s.replace(pos, toReplace.length(), replaceWith); + pos += replaceWith.length(); + } +} diff --git a/code/components/jomjol_helper/Helper.h b/code/components/jomjol_helper/Helper.h index 9cfdf702..e81d2fbd 100644 --- a/code/components/jomjol_helper/Helper.h +++ b/code/components/jomjol_helper/Helper.h @@ -98,6 +98,7 @@ const char* get404(void); std::string UrlDecode(const std::string& value); +void replaceAll(std::string& s, const std::string& toReplace, const std::string& replaceWith); bool replaceString(std::string& s, std::string const& toReplace, std::string const& replaceWith); bool replaceString(std::string& s, std::string const& toReplace, std::string const& replaceWith, bool logIt); bool isInString(std::string& s, std::string const& toFind); diff --git a/code/components/openmetrics/CMakeLists.txt b/code/components/openmetrics/CMakeLists.txt new file mode 100644 index 00000000..e2543939 --- /dev/null +++ b/code/components/openmetrics/CMakeLists.txt @@ -0,0 +1,7 @@ +FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*) + +idf_component_register(SRCS ${app_sources} + INCLUDE_DIRS "." + REQUIRES jomjol_image_proc) + + diff --git a/code/components/openmetrics/openmetrics.cpp b/code/components/openmetrics/openmetrics.cpp new file mode 100644 index 00000000..c90100e8 --- /dev/null +++ b/code/components/openmetrics/openmetrics.cpp @@ -0,0 +1,43 @@ +#include "openmetrics.h" + +/** + * create a singe metric from the given input + **/ +std::string createMetric(const std::string &metricName, const std::string &help, const std::string &type, const std::string &value) +{ + return "# HELP " + metricName + " " + help + "\n" + + "# TYPE " + metricName + " " + type + "\n" + + metricName + " " + value + "\n"; +} + +/** + * Generate the MetricFamily from all available sequences + * @returns the string containing the text wire format of the MetricFamily + **/ +std::string createSequenceMetrics(std::string prefix, const std::vector &numbers) +{ + std::string res; + + for (const auto &number : numbers) + { + // only valid data is reported (https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#missing-data) + if (number->ReturnValue.length() > 0) + { + auto label = number->name; + + // except newline, double quote, and backslash (https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#abnf) + // to keep it simple, these characters are just removed from the label + replaceAll(label, "\\", ""); + replaceAll(label, "\"", ""); + replaceAll(label, "\n", ""); + res += prefix + "_flow_value{sequence=\"" + label + "\"} " + number->ReturnValue + "\n"; + } + } + + // prepend metadata if a valid metric was created + if (res.length() > 0) + { + res = "# HELP " + prefix + "_flow_value current value of meter readout\n# TYPE " + prefix + "_flow_value gauge\n" + res; + } + return res; +} diff --git a/code/components/openmetrics/openmetrics.h b/code/components/openmetrics/openmetrics.h new file mode 100644 index 00000000..b75e98cd --- /dev/null +++ b/code/components/openmetrics/openmetrics.h @@ -0,0 +1,15 @@ +#pragma once + +#ifndef OPENMETRICS_H +#define OPENMETRICS_H + +#include +#include +#include + +#include "ClassFlowDefineTypes.h" + +std::string createMetric(const std::string &metricName, const std::string &help, const std::string &type, const std::string &value); +std::string createSequenceMetrics(std::string prefix, const std::vector &numbers); + +#endif // OPENMETRICS_H diff --git a/code/main/server_main.cpp b/code/main/server_main.cpp index b0a9a77e..7527614b 100644 --- a/code/main/server_main.cpp +++ b/code/main/server_main.cpp @@ -459,7 +459,7 @@ httpd_handle_t start_webserver(void) config.server_port = 80; config.ctrl_port = 32768; config.max_open_sockets = 5; //20210921 --> previously 7 - config.max_uri_handlers = 39; // previously 24, 20220511: 35, 20221220: 37, 2023-01-02:38 + config.max_uri_handlers = 40; // Make sure this fits all URI handlers. Memory usage in bytes: 6*max_uri_handlers config.max_resp_headers = 8; config.backlog_conn = 5; config.lru_purge_enable = true; // this cuts old connections if new ones are needed. diff --git a/code/test/components/openmetrics/test_openmetrics.cpp b/code/test/components/openmetrics/test_openmetrics.cpp new file mode 100644 index 00000000..00600cd4 --- /dev/null +++ b/code/test/components/openmetrics/test_openmetrics.cpp @@ -0,0 +1,65 @@ +#include +#include + +void test_createMetric() +{ + // simple happy path + const char *expected = "# HELP metric_name short description\n# TYPE metric_name gauge\nmetric_name 123.456\n"; + std::string result = createMetric("metric_name", "short description", "gauge", "123.456"); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); +} + +/** + * test the replaceString function as it's a dependency to sanitize sequence names + */ +void test_replaceString() +{ + std::string sample = "hello\\world\\"; + replaceAll(sample, "\\", ""); + TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str()); + + sample = "hello\"world\""; + replaceAll(sample, "\"", ""); + TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str()); + + sample = "hello\nworld\n"; + replaceAll(sample, "\n", ""); + TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str()); + + sample = "\\\\\\\\\\\\\\\\\\hello\\world\\\\\\\\\\\\\\\\\\\\"; + replaceAll(sample, "\\", ""); + TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str()); +} + +void test_createSequenceMetrics() +{ + std::vector NUMBERS; + NumberPost *number_1 = new NumberPost; + number_1->name = "main"; + number_1->ReturnValue = "123.456"; + NUMBERS.push_back(number_1); + + const std::string metricNamePrefix = "ai_on_the_edge_device"; + const std::string metricName = metricNamePrefix + "_flow_value"; + + std::string expected1 = "# HELP " + metricName + " current value of meter readout\n# TYPE " + metricName + " gauge\n" + + metricName + "{sequence=\"" + number_1->name + "\"} " + number_1->ReturnValue + "\n"; + TEST_ASSERT_EQUAL_STRING(expected1.c_str(), createSequenceMetrics(metricNamePrefix, NUMBERS).c_str()); + + NumberPost *number_2 = new NumberPost; + number_2->name = "secondary"; + number_2->ReturnValue = "1.0"; + NUMBERS.push_back(number_2); + + std::string expected2 = "# HELP " + metricName + " current value of meter readout\n# TYPE " + metricName + " gauge\n" + + metricName + "{sequence=\"" + number_1->name + "\"} " + number_1->ReturnValue + "\n" + + metricName + "{sequence=\"" + number_2->name + "\"} " + number_2->ReturnValue + "\n"; + TEST_ASSERT_EQUAL_STRING(expected2.c_str(), createSequenceMetrics(metricNamePrefix, NUMBERS).c_str()); +} + +void test_openmetrics() +{ + test_createMetric(); + test_replaceString(); + test_createSequenceMetrics(); +} diff --git a/code/test/test_suite_flowcontroll.cpp b/code/test/test_suite_flowcontroll.cpp index c59c7e96..d152f199 100644 --- a/code/test/test_suite_flowcontroll.cpp +++ b/code/test/test_suite_flowcontroll.cpp @@ -20,6 +20,7 @@ #include "components/jomjol-flowcontroll/test_PointerEvalAnalogToDigitNew.cpp" #include "components/jomjol-flowcontroll/test_getReadoutRawString.cpp" #include "components/jomjol-flowcontroll/test_cnnflowcontroll.cpp" +#include "components/openmetrics/test_openmetrics.cpp" #include "components/jomjol_mqtt/test_server_mqtt.cpp" bool Init_NVS_SDCard() @@ -167,6 +168,7 @@ extern "C" void app_main() // getReadoutRawString test RUN_TEST(test_getReadoutRawString); + RUN_TEST(test_openmetrics); RUN_TEST(test_mqtt); UNITY_END();