Trim app and recovery binaries

This commit is contained in:
Sebastien L
2023-10-11 12:36:17 -04:00
parent 9ebe717e74
commit 484d8c54a8
34 changed files with 1977 additions and 1126 deletions

View File

@@ -0,0 +1,124 @@
#define LOG_LOCAL_LEVEL ESP_LOG_VERBOSE
#include "Batch.h"
#include "esp_event.h"
#include "esp_http_client.h"
#include "esp_log.h"
#include "esp_netif.h"
#include "esp_ota_ops.h"
#include "esp_tls.h"
#include "nvs_flash.h"
#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
#include "esp_crt_bundle.h"
#endif
#include "esp_system.h"
#include "http_handlers.h"
#include "nvs.h"
#include "nvs_flash.h"
#include "nvs_utilities.h"
#include "tools.h"
#include <algorithm>
#include <iomanip>
#include <sstream>
#include <string>
#include <sys/param.h>
#if CONFIG_WITH_METRICS
static const char* const TAG = "MetricsBatch";
static const char* const feature_evt_name = "$feature_flag_called";
static const char* const feature_flag_name = "$feature_flag";
static const char* const feature_flag_response_name = "$feature_flag_response";
namespace Metrics {
Event& Batch::add_feature_event() { return add_event(feature_evt_name); }
void Batch::add_remove_feature_event(const char* name, bool active) {
if (!active) {
remove_feature_event(name);
} else {
add_event(feature_evt_name).add_property(feature_flag_name, name);
}
}
Event& Batch::add_feature_variant_event(const char* const name, const char* const value) {
return add_event(feature_evt_name)
.add_property(feature_flag_name, name)
.add_property(feature_flag_response_name, value);
}
void Batch::remove_feature_event(const char* name) {
for (Metrics::Event& e : _events) {
if (strcmp(e.get_name(), feature_evt_name) == 0) {
e.remove_property(feature_flag_name, name);
return;
}
}
}
cJSON* Batch::to_json() {
cJSON* batch_json = cJSON_CreateArray();
for (Metrics::Event& e : _events) {
cJSON_AddItemToArray(batch_json, e.to_json(_metrics_uid.c_str()));
}
cJSON* message = cJSON_CreateObject();
cJSON_AddItemToObject(message, "batch", batch_json);
cJSON_AddStringToObject(message, "api_key", _api_key);
return batch_json;
}
char* Batch::to_json_str() {
cJSON* json = to_json();
char* json_str = cJSON_PrintUnformatted(json);
cJSON_Delete(json);
return json_str;
}
void Batch::push() {
int status_code = 0;
if (_metrics_uid.empty() && !_warned) {
ESP_LOGW(TAG, "Metrics disabled; no CID found");
_warned = true;
return;
}
char* json_str = to_json_str();
ESP_LOGV(TAG, "Metrics payload: %s", json_str);
time_t start_time = millis();
status_code = metrics_http_post_request(json_str, _url);
if (status_code == 200 || status_code == 204) {
_events.clear();
}
FREE_AND_NULL(json_str)
ESP_LOGD(TAG, "Total duration for metrics call: %lu. ", millis() - start_time);
}
void Batch::build_guid() {
uint8_t raw[16];
std::ostringstream oss;
esp_fill_random(raw, 16);
std::for_each(std::begin(raw), std::end(raw), [&oss](const uint8_t& byte) {
oss << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(byte);
});
_metrics_uid = oss.str();
}
void Batch::assign_id() {
size_t size = 0;
esp_err_t esp_err = ESP_OK;
_metrics_uid = std::string((char*)get_nvs_value_alloc_for_partition(
NVS_DEFAULT_PART_NAME, TAG, NVS_TYPE_BLOB, "cid", &size));
if (_metrics_uid[0] == 'G') {
ESP_LOGW(TAG, "Invalid ID. %s", _metrics_uid.c_str());
_metrics_uid.clear();
}
if (_metrics_uid.empty()) {
build_guid();
if (_metrics_uid.empty()) {
ESP_LOGE(TAG, "ID Failed");
return;
}
ESP_LOGW(TAG, "Metrics ID: %s", _metrics_uid.c_str());
esp_err = store_nvs_value_len_for_partition(NVS_DEFAULT_PART_NAME, TAG, NVS_TYPE_BLOB,
"cid", _metrics_uid.c_str(), _metrics_uid.length() + 1);
if (esp_err != ESP_OK) {
ESP_LOGE(TAG, "Store ID failed: %s", esp_err_to_name(esp_err));
}
}
}
} // namespace Metrics
#endif

View File

@@ -0,0 +1,46 @@
#pragma once
#include "Events.h"
#include <string>
#ifdef __cplusplus
namespace Metrics {
extern "C" {
#endif
#ifdef __cplusplus
class Batch {
private:
std::list<Event> _events;
bool _warned = false;
std::string _metrics_uid = nullptr;
const char* _api_key = nullptr;
const char* _url = nullptr;
void build_guid();
void assign_id();
public:
Batch() = default;
void configure(const char* api_key, const char* url) {
_api_key = api_key;
_url = url;
assign_id();
}
Event& add_feature_event();
void add_remove_feature_event(const char* name, bool active);
Event& add_feature_variant_event(const char* const name, const char* const value);
Event& add_event(const char* name) {
_events.emplace_back(name);
return _events.back();
}
bool has_events() const { return !_events.empty(); }
void remove_feature_event(const char* name);
cJSON* to_json();
char* to_json_str();
void push();
};
}
#endif
#ifdef __cplusplus
}
#endif

View File

@@ -0,0 +1,5 @@
idf_component_register(SRC_DIRS .
INCLUDE_DIRS .
REQUIRES json tools platform_config wifi-manager esp-tls platform_config
PRIV_REQUIRES esp32 freertos
)

View File

@@ -0,0 +1,98 @@
#include "Events.h"
#include <algorithm>
#include "esp_app_format.h"
#include "esp_ota_ops.h"
#if CONFIG_WITH_METRICS
static const char* const TAG = "MetricsEvent";
namespace Metrics {
Event& Event::add_property(const char* name, const char* value) {
ESP_LOGV(TAG, "Adding property %s:%s to event %s",name,value,_name);
char* mutable_name = strdup_psram(name); // Cast away const-ness, be careful with this
auto elem = properties.find(mutable_name);
FREE_AND_NULL(mutable_name)
if (elem == properties.end()) {
ESP_LOGV(TAG, "Adding property %s:%s to event %s",name,value,_name);
properties.insert({strdup_psram(name), strdup_psram(value)});
} else {
ESP_LOGV(TAG, "Replacing value for property %s. Old: %s New: %s, Event: %s",name,elem->second,value,name);
FREE_AND_NULL(elem->second)
elem->second = strdup_psram(value);
}
return *this;
}
bool Event::has_property_value(const char* name, const char* value) const {
ESP_LOGV(TAG, "Checking if event %s property %s has value %s",_name, name,value);
return std::any_of(properties.begin(), properties.end(),
[name, value](const std::pair<const char* const, char*>& kv) {
ESP_LOGV(TAG, "Found property %s=%s", name,value);
return strcmp(kv.first, name) == 0 && strcmp(kv.second, value) == 0;
});
}
void Event::remove_property(const char* name, const char* value) {
auto it = properties.begin();
ESP_LOGV(TAG, "Removing event %s property %s=%s",_name, name,value);
while (it != properties.end()) {
if (strcmp(it->first, name) == 0 && strcmp(it->second, value)) {
properties.erase(it);
return;
}
}
ESP_LOGV(TAG, "Property %s=%s not found.", name,value);
}
cJSON* Event::properties_to_json() {
ESP_LOGV(TAG, "Event %s properties to json.",_name);
const esp_app_desc_t* desc = esp_ota_get_app_description();
#ifdef CONFIG_FW_PLATFORM_NAME
const char* platform = CONFIG_FW_PLATFORM_NAME;
#else
const char* platform = desc->project_name;
#endif
cJSON* prop_json = cJSON_CreateObject();
auto it = properties.begin();
while (it != properties.end()) {
cJSON_AddStringToObject(prop_json, it->first, it->second);
++it;
}
cJSON_AddStringToObject(prop_json, "platform", platform);
cJSON_AddStringToObject(prop_json, "build", desc->version);
dump_json_content("User properties for event:", prop_json, ESP_LOG_VERBOSE);
return prop_json;
}
cJSON* Event::to_json(const char* distinct_id) {
// The target structure looks like this
// {
// "event": "batched_event_name_1",
// "properties": {
// "distinct_id": "user distinct id",
// "account_type": "pro"
// },
// "timestamp": "[optional timestamp in ISO 8601 format]"
// }
ESP_LOGV(TAG,"Event %s to json",_name);
free_json();
_json = cJSON_CreateObject();
cJSON_AddStringToObject(_json, "name", _name);
cJSON_AddItemToObject(_json, "properties", properties_to_json());
char buf[26] = {};
strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&_time));
// this will work too, if your compiler doesn't support %F or %T:
// strftime(buf, sizeof buf, "%Y-%m-%dT%H:%M:%SZ", gmtime(&now));
cJSON_AddStringToObject(_json, "timestamp", buf);
cJSON* prop_json = properties_to_json();
cJSON_AddStringToObject(prop_json, "distinct_id", distinct_id);
dump_json_content("Full Event:", _json, ESP_LOG_VERBOSE);
return _json;
}
void Event::free_json() { cJSON_Delete(_json); }
void Event::update_time() {
if (_time == 0) {
_time = time(nullptr);
}
}
} // namespace Metrics
#endif

View File

@@ -0,0 +1,53 @@
#pragma once
#ifdef __cplusplus
#include "esp_log.h"
#include "tools.h"
#include <cJSON.h>
#include <ctime>
#include <list>
#include <map>
#include <stdio.h>
#include <string.h>
#include <string>
namespace Metrics {
struct StrCompare {
bool operator()(const char* a, const char* b) const { return strcmp(a, b) < 0; }
};
class Event {
public:
std::map<char*, char*, StrCompare> properties;
Event& add_property(const char* name, const char* value);
bool has_property_value(const char* name, const char* value) const;
void remove_property(const char* name, const char* value);
cJSON* properties_to_json();
cJSON* to_json(const char* distinct_id);
void free_json();
void update_time();
explicit Event(const char* name) {
_name = strdup_psram(name);
memset(&_time, 0x00, sizeof(_time));
}
const char* get_name() const { return _name; }
~Event() {
FREE_AND_NULL(_name);
// Iterate through the map and free the elements
for (auto& kv : properties) {
free((void*)kv.first);
free(kv.second);
}
properties.clear(); // Clear the map after freeing memory
FREE_AND_NULL(_json);
}
private:
char* _name = nullptr;
time_t _time;
cJSON* _json = nullptr;
};
} // namespace Metrics
#endif

View File

@@ -0,0 +1,148 @@
#define LOG_LOCAL_LEVEL ESP_LOG_VERBOSE
#include "Metrics.h"
#include "Batch.h"
#include "esp_event.h"
#include "esp_log.h"
#include "esp_netif.h"
#include "esp_ota_ops.h"
#include "esp_system.h"
#include "esp_tls.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "nvs_flash.h"
#include "tools.h"
#include <cstdarg>
#include <cstdio>
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <vector>
#include "cJSON.h"
#include "freertos/timers.h"
#include "network_manager.h"
#include "platform_config.h"
static const char* TAG = "metrics";
#if CONFIG_WITH_METRICS
extern bool is_network_connected();
#define METRICS_CLIENT_ID_LEN 50
#define MAX_HTTP_RECV_BUFFER 512
static bool metrics_usage_gen = false;
static time_t metrics_usage_gen_time = 0;
#ifndef METRICS_API_KEY
#pragma message "Metrics API key needs to be passed from the environment"
#define METRICS_API_KEY "ZZZ"
#endif
static const char* metrics_api_key =
static const char* parms_str = "params";
static const char* properties_str = "properties";
static const char* user_properties_str = "user_properties";
static const char* items_str = "items";
static const char* quantity_str = "quantity";
static const char* metrics_url = "https://app.posthog.com";
static TimerHandle_t timer;
extern cJSON* get_cmd_list();
Metrics::Batch batch;
static void metrics_timer_cb(void* timer_id) {
if (batch.has_events()) {
if (!is_network_connected()) {
ESP_LOGV(TAG, "Network not connected. can't flush");
} else {
ESP_LOGV(TAG, "Pushing events");
batch.push();
}
}
if (millis() > metrics_usage_gen_time && !metrics_usage_gen) {
metrics_usage_gen = true;
ESP_LOGV(TAG, "Generate command list to pull features");
cJSON* cmdlist = get_cmd_list();
dump_json_content("generated cmd list", cmdlist, ESP_LOG_VERBOSE);
cJSON_Delete(cmdlist);
}
}
void metrics_init() {
ESP_LOGV(TAG, "Initializing metrics");
batch.configure(metrics_api_key, metrics_url);
if (!timer) {
ESP_LOGE(TAG, "Metrics Timer failure");
} else {
ESP_LOGV(TAG, "Starting timer");
xTimerStart(timer, portMAX_DELAY);
}
// set a 20 seconds delay before generating the
// features so the system has time to boot
metrics_usage_gen_time = millis() + 20000;
}
void metrics_event_playback(const char* source) {
ESP_LOGV(TAG, "Playback event: %s", source);
auto event = batch.add_event("play").add_property("source", source);
}
void metrics_event_boot(const char* partition) {
ESP_LOGV(TAG, "Boot event %s", partition);
auto event = batch.add_event("start");
event.add_property("partition", partition);
}
void metrics_add_feature_variant(const char* name, const char* format, ...) {
va_list args;
ESP_LOGV(TAG, "Feature %s", name);
va_start(args, format);
// Determine the required buffer size
int size = vsnprintf(nullptr, 0, format, args);
va_end(args); // Reset the va_list
// Allocate buffer and format the string
std::vector<char> buffer(size + 1); // +1 for the null-terminator
va_start(args, format);
vsnprintf(buffer.data(), buffer.size(), format, args);
va_end(args);
// Now buffer.data() contains the formatted string
batch.add_feature_variant_event(name, buffer.data());
}
void metrics_add_feature(const char* name, bool active) {
ESP_LOGV(TAG, "Adding feature %s: %s", name, active ? "ACTIVE" : "INACTIVE");
batch.add_remove_feature_event(name, active);
}
void metrics_event(const char* name) {
ESP_LOGV(TAG, "Adding Event %s", name);
batch.add_event(name);
}
#else
static const char * not_enabled = " - (metrics not enabled, this is just marking where the call happens)";
void metrics_init(){
#pragma message("Metrics disabled")
ESP_LOGD(TAG,"Metrics init%s",not_enabled);
}
void metrics_event_boot(const char* partition){
ESP_LOGD(TAG,"Metrics Event Boot from partition %s%s",partition,not_enabled);
}
void metrics_event(const char* name){
ESP_LOGD(TAG,"Metrics Event %s%s",name,not_enabled);
}
void metrics_add_feature(const char* name, bool active) {
ESP_LOGD(TAG,"Metrics add feature %s%s%s",name,active?"ACTIVE":"INACTIVE",not_enabled);
}
void metrics_add_feature_variant(const char* name, const char* format, ...){
va_list args;
ESP_LOGV(TAG, "Feature %s", name);
va_start(args, format);
// Determine the required buffer size
int size = vsnprintf(nullptr, 0, format, args);
va_end(args); // Reset the va_list
// Allocate buffer and format the string
std::vector<char> buffer(size + 1); // +1 for the null-terminator
va_start(args, format);
vsnprintf(buffer.data(), buffer.size(), format, args);
va_end(args);
ESP_LOGD(TAG,"Metrics add feature %s variant %s%s",name,buffer.data(),not_enabled);
}
#endif

View File

@@ -0,0 +1,18 @@
#pragma once
#include <stdbool.h>
#include <stdlib.h>
#ifdef __cplusplus
extern "C" {
#endif
void metrics_event_playback(const char* source);
void metrics_event_boot(const char* partition);
void metrics_event(const char* name);
void metrics_add_feature(const char* name, bool active);
void metrics_add_feature_variant(const char* name, const char* format, ...);
void metrics_init();
void metrics_flush();
#ifdef __cplusplus
}
#endif

View File

@@ -0,0 +1,163 @@
#include "http_handlers.h"
#include "esp_http_client.h"
#include "esp_log.h"
#include "esp_tls.h"
#include "tools.h"
#include <sys/param.h>
#if CONFIG_WITH_METRICS
static const char* TAG = "metrics_http";
static char* output_buffer; // Buffer to store response of http request from
// event handler
static int output_len = 0; // Stores number of bytes read
#define MAX_HTTP_OUTPUT_BUFFER 2048
// Common function signature for event handlers
typedef void (*HttpEventHandler)(esp_http_client_event_t* evt);
static void handle_http_error(esp_http_client_event_t* evt) { ESP_LOGV(TAG, "ERROR"); }
static void handle_http_connected(esp_http_client_event_t* evt) {
ESP_LOGV(TAG, "ON_CONNECTED");
}
static void handle_http_header_sent(esp_http_client_event_t* evt) {
ESP_LOGV(TAG, "HEADER_SENT");
}
static void handle_http_on_header(esp_http_client_event_t* evt) {
ESP_LOGV(TAG, "ON_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);
}
static void handle_http_on_data(esp_http_client_event_t* evt) {
ESP_LOGV(TAG, "ON_DATA, len=%d", evt->data_len);
ESP_LOGV(TAG, "ON_DATA, len=%d", evt->data_len);
// Clean the buffer in case of a new request
if (output_len == 0 && evt->user_data) {
// we are just starting to copy the output data into the use
ESP_LOGV(TAG, "Resetting buffer");
memset(evt->user_data, 0, MAX_HTTP_OUTPUT_BUFFER);
}
/*
* Check for chunked encoding is added as the URL for chunked encoding used in this example
* returns binary data. However, event handler can also be used in case chunked encoding is
* used.
*/
// If user_data buffer is configured, copy the response into the buffer
int copy_len = 0;
if (evt->user_data) {
ESP_LOGV(TAG, "Not Chunked response, with user data");
// The last byte in evt->user_data is kept for the NULL character in
// case of out-of-bound access.
copy_len = MIN(evt->data_len, (MAX_HTTP_OUTPUT_BUFFER - output_len));
if (copy_len) {
memcpy(evt->user_data + output_len, evt->data, copy_len);
}
} else {
int content_len = esp_http_client_get_content_length(evt->client);
if (esp_http_client_is_chunked_response(evt->client)) {
esp_http_client_get_chunk_length(evt->client, &content_len);
}
if (output_buffer == NULL) {
// We initialize output_buffer with 0 because it is used by
// strlen() and similar functions therefore should be null
// terminated.
size_t len=(content_len + 1) * sizeof(char);
ESP_LOGV(TAG, "Init buffer %d",len);
output_buffer = (char*)malloc_init_external(len);
output_len = 0;
if (output_buffer == NULL) {
ESP_LOGE(TAG, "Buffer alloc failed.");
return;
}
}
copy_len = MIN(evt->data_len, (content_len - output_len));
if (copy_len) {
memcpy(output_buffer + output_len, evt->data, copy_len);
}
}
output_len += copy_len;
}
static void handle_http_on_finish(esp_http_client_event_t* evt) {
ESP_LOGD(TAG, "ON_FINISH");
if (output_buffer != NULL) {
ESP_LOGV(TAG, "Response: %s", output_buffer);
free(output_buffer);
output_buffer = NULL;
}
output_len = 0;
}
static void handle_http_disconnected(esp_http_client_event_t* evt) {
ESP_LOGI(TAG, "DISCONNECTED");
int mbedtls_err = 0;
esp_err_t err =
esp_tls_get_and_clear_last_error((esp_tls_error_handle_t)evt->data, &mbedtls_err, NULL);
if (err != 0) {
ESP_LOGI(TAG, "Last error : %s", esp_err_to_name(err));
ESP_LOGI(TAG, "Last mbedtls err 0x%x", mbedtls_err);
}
if (output_buffer != NULL) {
free(output_buffer);
output_buffer = NULL;
}
output_len = 0;
}
static const HttpEventHandler eventHandlers[] = {
handle_http_error, // HTTP_EVENT_ERROR
handle_http_connected, // HTTP_EVENT_ON_CONNECTED
handle_http_header_sent, // HTTP_EVENT_HEADERS_SENT
handle_http_header_sent, // HTTP_EVENT_HEADER_SENT (alias for HTTP_EVENT_HEADERS_SENT)
handle_http_on_header, // HTTP_EVENT_ON_HEADER
handle_http_on_data, // HTTP_EVENT_ON_DATA
handle_http_on_finish, // HTTP_EVENT_ON_FINISH
handle_http_disconnected // HTTP_EVENT_DISCONNECTED
};
esp_err_t metrics_http_event_handler(esp_http_client_event_t* evt) {
if (evt->event_id < 0 || evt->event_id >= sizeof(eventHandlers) / sizeof(eventHandlers[0])) {
ESP_LOGE(TAG, "Invalid event ID: %d", evt->event_id);
return ESP_FAIL;
}
eventHandlers[evt->event_id](evt);
return ESP_OK;
}
int metrics_http_post_request(const char* payload, const char* url) {
int status_code = 0;
esp_http_client_config_t config = {.url = url,
.disable_auto_redirect = false,
.event_handler = metrics_http_event_handler,
.transport_type = HTTP_TRANSPORT_OVER_SSL,
.user_data = NULL, // local_response_buffer, // Pass address of
// local buffer to get response
.skip_cert_common_name_check = true
};
esp_http_client_handle_t client = esp_http_client_init(&config);
esp_err_t err = esp_http_client_set_method(client, HTTP_METHOD_POST);
if (err == ESP_OK) {
err = esp_http_client_set_header(client, "Content-Type", "application/json");
}
if (err == ESP_OK) {
ESP_LOGV(TAG, "Setting payload: %s", payload);
err = esp_http_client_set_post_field(client, payload, strlen(payload));
}
if (err == ESP_OK) {
err = esp_http_client_perform(client);
}
if (err == ESP_OK) {
status_code = esp_http_client_get_status_code(client);
ESP_LOGD(TAG, "metrics call Status = %d, content_length = %d",
esp_http_client_get_status_code(client), esp_http_client_get_content_length(client));
} else {
status_code = 500;
ESP_LOGW(TAG, "metrics call Status failed: %s", esp_err_to_name(err));
}
esp_http_client_cleanup(client);
return status_code;
}
#endif

View File

@@ -0,0 +1,11 @@
#pragma once
#ifdef __cplusplus
extern "C" {
#endif
int metrics_http_post_request(const char* payload, const char* url);
#ifdef __cplusplus
}
#endif