Implement a camera livestream handler (#2286)

code/components/jomjol_controlcamera/ClassControllCamera.cpp

@@ -32,6 +32,14 @@
 
 static const char *TAG = "CAM"; 
 
+
+/* Camera live stream */
+#define PART_BOUNDARY "123456789000000000000987654321"
+static const char* _STREAM_CONTENT_TYPE = "multipart/x-mixed-replace;boundary=" PART_BOUNDARY;
+static const char* _STREAM_BOUNDARY = "\r\n--" PART_BOUNDARY "\r\n";
+static const char* _STREAM_PART = "Content-Type: image/jpeg\r\nContent-Length: %u\r\n\r\n";
+
+
 static camera_config_t camera_config = {
     .pin_pwdn = CAM_PIN_PWDN,
     .pin_reset = CAM_PIN_RESET,
@@ -521,6 +529,74 @@ esp_err_t CCamera::CaptureToHTTP(httpd_req_t *req, int delay)
 }
 
 
+esp_err_t CCamera::CaptureToStream(httpd_req_t *req, bool FlashlightOn)
+{
+    esp_err_t res = ESP_OK;
+    size_t fb_len = 0;
+    int64_t fr_start;
+    char * part_buf[64];
+
+    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Live stream started");
+
+    if (FlashlightOn) {
+        LEDOnOff(true);
+        LightOnOff(true);
+    }
+
+    //httpd_resp_set_hdr(req, "Access-Control-Allow-Origin", "*");  //stream is blocking web interface, only serving to local
+
+    httpd_resp_set_type(req, _STREAM_CONTENT_TYPE);
+    httpd_resp_send_chunk(req, _STREAM_BOUNDARY, strlen(_STREAM_BOUNDARY));
+
+    while(1)
+    {
+        fr_start = esp_timer_get_time();
+        camera_fb_t *fb = esp_camera_fb_get();
+        esp_camera_fb_return(fb);
+        fb = esp_camera_fb_get();
+        if (!fb) {
+            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "CaptureToStream: Camera framebuffer not available");
+            break;
+        }
+        fb_len = fb->len;
+   
+        if (res == ESP_OK){
+            size_t hlen = snprintf((char *)part_buf, sizeof(part_buf), _STREAM_PART, fb_len);
+            res = httpd_resp_send_chunk(req, (const char *)part_buf, hlen);
+        }
+        if (res == ESP_OK){
+            res = httpd_resp_send_chunk(req, (const char *)fb->buf, fb_len);
+        }
+        if (res == ESP_OK){
+            res = httpd_resp_send_chunk(req, _STREAM_BOUNDARY, strlen(_STREAM_BOUNDARY));
+        }
+        
+        esp_camera_fb_return(fb);
+
+        int64_t fr_end = esp_timer_get_time();
+        ESP_LOGD(TAG, "JPG: %uKB %ums", (uint32_t)(fb_len/1024), (uint32_t)((fr_end - fr_start)/1000));
+
+        if (res != ESP_OK){ // Exit loop, e.g. also when closing the webpage
+            break;
+        }
+
+        int64_t fr_delta_ms = (fr_end - fr_start) / 1000;
+        if (CAM_LIVESTREAM_REFRESHRATE > fr_delta_ms) {
+            const TickType_t xDelay = (CAM_LIVESTREAM_REFRESHRATE - fr_delta_ms)  / portTICK_PERIOD_MS;
+            ESP_LOGD(TAG, "Stream: sleep for: %ldms", (long) xDelay*10);
+            vTaskDelay(xDelay);        
+        }
+    }
+
+    LEDOnOff(false);
+    LightOnOff(false);
+
+    LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Live stream stopped");
+
+    return res;
+}
+
+
 void CCamera::LightOnOff(bool status)
 {
     GpioHandler* gpioHandler = gpio_handler_get();

code/components/jomjol_controlcamera/ClassControllCamera.h

@@ -40,6 +40,7 @@ class CCamera {
         void LightOnOff(bool status);
         void LEDOnOff(bool status);
         esp_err_t CaptureToHTTP(httpd_req_t *req, int delay = 0);
+        esp_err_t CaptureToStream(httpd_req_t *req, bool FlashlightOn);
         void SetQualitySize(int qual, framesize_t resol);
         bool SetBrightnessContrastSaturation(int _brightness, int _contrast, int _saturation);
         void GetCameraParameter(httpd_req_t *req, int &qual, framesize_t &resol);

code/components/jomjol_flowcontroll/MainFlowControl.cpp

@@ -208,6 +208,40 @@ esp_err_t handler_init(httpd_req_t *req)
 }
 
 
+esp_err_t handler_stream(httpd_req_t *req)
+{
+    #ifdef DEBUG_DETAIL_ON      
+        LogFile.WriteHeapInfo("handler_stream - Start");       
+        ESP_LOGD(TAG, "handler_stream uri: %s", req->uri);
+    #endif
+
+    char _query[50];
+    char _value[10];
+    bool flashlightOn = false;
+
+    if (httpd_req_get_url_query_str(req, _query, 50) == ESP_OK)
+    {
+//        ESP_LOGD(TAG, "Query: %s", _query);
+        if (httpd_query_key_value(_query, "flashlight", _value, 10) == ESP_OK)
+        {
+            #ifdef DEBUG_DETAIL_ON       
+                ESP_LOGD(TAG, "flashlight is found%s", _size);
+            #endif
+            if (strlen(_value) > 0)
+                flashlightOn = true;
+        }
+    }
+
+    Camera.CaptureToStream(req, flashlightOn);
+
+    #ifdef DEBUG_DETAIL_ON      
+        LogFile.WriteHeapInfo("handler_stream - Done");       
+    #endif
+
+    return ESP_OK;
+}
+
+
 esp_err_t handler_flow_start(httpd_req_t *req) {
 
     #ifdef DEBUG_DETAIL_ON          
@@ -1094,4 +1128,10 @@ void register_server_main_flow_task_uri(httpd_handle_t server)
     camuri.handler   = handler_get_heap;
     camuri.user_ctx  = (void*) "Heap"; 
     httpd_register_uri_handler(server, &camuri);
+
+    camuri.uri       = "/stream";
+    camuri.handler   = handler_stream;
+    camuri.user_ctx  = (void*) "stream"; 
+    httpd_register_uri_handler(server, &camuri);
+
 }

code/include/defines.h

@@ -55,39 +55,48 @@
 
     //compiler optimization for tflite-micro-esp-examples
     #define XTENSA
-    //#define CONFIG_IDF_TARGET_ARCH_XTENSA //not needed with platformio/espressif32 @ 5.2.0
+    //#define CONFIG_IDF_TARGET_ARCH_XTENSA     //not needed with platformio/espressif32 @ 5.2.0
 
 
-    //ClassControllCamera + ClassFlowTakeImage + connect_wlan + main
+    //Statusled + ClassControllCamera
-    #define FLASH_GPIO GPIO_NUM_4
+    #define BLINK_GPIO GPIO_NUM_33              // PIN for red board LED
-    #define BLINK_GPIO GPIO_NUM_33
 
-    //interface_mqtt + read_wlanini
-    #define __HIDE_PASSWORD
 
     //ClassControllCamera
-    #define USE_PWM_LEDFLASH // if __LEDGLOBAL is defined, a global variable is used for LED control, otherwise locally and each time a new
+    #define FLASH_GPIO GPIO_NUM_4               // PIN for flashlight LED
+    #define USE_PWM_LEDFLASH                    // if __LEDGLOBAL is defined, a global variable is used for LED control, otherwise locally and each time a new
+    #define CAM_LIVESTREAM_REFRESHRATE 500      // Camera livestream feature: Waiting time in milliseconds to refresh image
 
-    //server_GPIO
-    #define __LEDGLOBAL
 
     //ClassControllCamera + ClassFlowTakeImage
     #define CAMERA_MODEL_AI_THINKER
     #define BOARD_ESP32CAM_AITHINKER
 
+
+    //server_GPIO
+    #define __LEDGLOBAL
+
+
     //server_GPIO + server_file + SoftAP
     #define CONFIG_FILE "/sdcard/config/config.ini"
     #define CONFIG_FILE_BACKUP "/sdcard/config/config.bak"
 
+
+    //interface_mqtt + read_wlanini
+    #define __HIDE_PASSWORD
+
+
     //ClassFlowControll + Main + SoftAP
     #define WLAN_CONFIG_FILE "/sdcard/wlan.ini"
 
+
     //main
     #define __SD_USE_ONE_LINE_MODE__
 
     // server_file + Helper
      #define FILE_PATH_MAX (255) //Max length a file path can have on storage
     
+
     //server_file +(ota_page.html + upload_script.html)
     #define MAX_FILE_SIZE   (8000*1024) // 8 MB Max size of an individual file. Make sure this value is same as that set in upload_script.html and ota_page.html!
     #define MAX_FILE_SIZE_STR "8MB"
@@ -98,37 +107,45 @@
     #define SERVER_HELPER_SCRATCH_BUFSIZE  8192
     #define SERVER_OTA_SCRATCH_BUFSIZE  1024 
 
+
     //server_file + server_help
     #define IS_FILE_EXT(filename, ext) \
     (strcasecmp(&filename[strlen(filename) - sizeof(ext) + 1], ext) == 0)
 
+
     //server_ota
     #define HASH_LEN 32 // SHA-256 digest length
     #define OTA_URL_SIZE 256
 
+
     //ClassFlow + ClassFlowImage + server_tflite
     #define LOGFILE_TIME_FORMAT "%Y%m%d-%H%M%S"
     #define LOGFILE_TIME_FORMAT_DATE_EXTR substr(0, 8)
     #define LOGFILE_TIME_FORMAT_HOUR_EXTR substr(9, 2)
 
+
     //ClassFlowControll
     #define READOUT_TYPE_VALUE 0
     #define READOUT_TYPE_PREVALUE 1
     #define READOUT_TYPE_RAWVALUE 2
     #define READOUT_TYPE_ERROR 3
 
+
     //ClassFlowControll: Serve alg_roi.jpg from memory as JPG
     #define ALGROI_LOAD_FROM_MEM_AS_JPG // Load ALG_ROI.JPG as rendered JPG from RAM
 
+
     //ClassFlowMQTT
     #define LWT_TOPIC        "connection"
     #define LWT_CONNECTED    "connected"
     #define LWT_DISCONNECTED "connection lost"
 
+
     //ClassFlowPostProcessing
     #define PREVALUE_TIME_FORMAT_OUTPUT "%Y-%m-%dT%H:%M:%S%z"
     #define PREVALUE_TIME_FORMAT_INPUT "%d-%d-%dT%d:%d:%d"
 
+
     //CImageBasis
     #define HTTP_BUFFER_SENT 1024
     #define MAX_JPG_SIZE 128000
@@ -139,14 +156,17 @@
     //#define STB_IMAGE_RESIZE_IMPLEMENTATION
     #define STBI_ONLY_JPEG // (save 2% of Flash, but breaks the alignment mark generation, see https://github.com/jomjol/AI-on-the-edge-device/issues/1721)
 
+
     //interface_influxdb
     #define MAX_HTTP_OUTPUT_BUFFER 2048
 
+
     //server_mqtt
     #define LWT_TOPIC        "connection"
     #define LWT_CONNECTED    "connected"
     #define LWT_DISCONNECTED "connection lost"
 
+
     //CTfLiteClass
     #define TFLITE_MINIMAL_CHECK(x)                              \
         if (!(x)) {                                                \

code/main/server_main.cpp

@@ -431,7 +431,7 @@ httpd_handle_t start_webserver(void)
     config.server_port = 80;
     config.ctrl_port = 32768;
     config.max_open_sockets = 5; //20210921 --> previously 7   
-    config.max_uri_handlers = 38; // previously 24, 20220511: 35, 20221220: 37, 2023-01-02:38             
+    config.max_uri_handlers = 39; // previously 24, 20220511: 35, 20221220: 37, 2023-01-02:38             
     config.max_resp_headers = 8;                        
     config.backlog_conn = 5;                        
     config.lru_purge_enable = true; // this cuts old connections if new ones are needed.