From 7a9955477a6a8eeff83dc4c9a0dbcfde201f8aa3 Mon Sep 17 00:00:00 2001
From: SybexX <Heinrich-Tuning@web.de>
Date: Sun, 23 Feb 2025 17:38:32 +0100
Subject: [PATCH] add data export (CSV files) (#3537)

* Update backup.html

* Add files via upload

* Update data_export.html

---------

Co-authored-by: CaCO3 <caco3@ruinelli.ch>
---
 sd-card/html/backup.html      | 444 +++++++++++++++++-----------------
 sd-card/html/data_export.html | 207 ++++++++++++++++
 sd-card/html/index.html       |   1 +
 3 files changed, 430 insertions(+), 222 deletions(-)
 create mode 100644 sd-card/html/data_export.html
diff --git a/sd-card/html/backup.html b/sd-card/html/backup.html
index 04af87f3..cc3c57c1 100644
--- a/sd-card/html/backup.html
+++ b/sd-card/html/backup.html
@@ -1,222 +1,222 @@
-<!DOCTYPE html>
-<html lang="en" xml:lang="en"> 
-<head>
-    <title>Backup/Restore Configuration</title>
-    <meta charset="UTF-8" />
-
-    <style>
-        h1 {font-size: 2em;}
-        h2 {font-size: 1.5em; margin-block-start: 0.0em; margin-block-end: 0.2em;}
-        h3 {font-size: 1.2em;}
-        p {font-size: 1em;}
-
-        input[type=number] {
-            width: 138px;
-            padding: 10px 5px;
-            display: inline-block;
-            border: 1px solid #ccc;
-            font-size: 16px; 
-        }
-
-        .button {
-            padding: 5px 10px;
-            width: 205px;
-            font-size: 16px;
-        }
-    </style>
-
-</head>
-
-<body style="font-family: arial; padding: 0px 10px;">
-    <h2>Backup Configuration</h2>
-    <p>With the following action the <a href="/fileserver/config/" target="_self">config</a> folder on the SD-card gets zipped and provided as a download.</p>
-
-    <button class="button" id="startBackup" type="button" onclick="startBackup()">Create Backup</button>
-    <p id=progress></p>
-    <hr>
-    <h2>Restore Configuration</h2>
-    <p>Use the <a href="/fileserver/config/" target="_self">File Server</a> to upload individual files.</p>
-</body>
-
-
-<script type="text/javascript" src="common.js?v=$COMMIT_HASH"></script>
-<script type="text/javascript" src="jszip.min.js?v=$COMMIT_HASH"></script>
-<script type="text/javascript" src="FileSaver.min.js?v=$COMMIT_HASH"></script>
-<script>
-
-function startBackup() {  
-    document.getElementById("progress").innerHTML = "Creating backup...<br>\n";
-    
-    // Get hostname
-    try {
-        var xhttp = new XMLHttpRequest();
-        xhttp.open("GET", getDomainname() + "/info?type=Hostname", false);
-        xhttp.send();
-        hostname = xhttp.responseText;
-    }
-    catch(err) {
-        setStatus("<span style=\"color: red\">Failed to fetch hostname: " + err.message + "!</span>");
-        return;
-    }
-    
-    // get date/time
-    var dateTime = new Date().toJSON().slice(0,10) + "_" + new Date().toJSON().slice(11,19).replaceAll(":", "-");
-    
-    zipFilename = hostname + "_" + dateTime + ".zip";
-    console.log(zipFilename);
-
-    // Get files list
-    setStatus("Fetching File List...");
-    try {
-        var xhttp = new XMLHttpRequest();
-        xhttp.open("GET", getDomainname() + "/fileserver/config/", false);
-        xhttp.send();
-        
-        var parser = new DOMParser();
-        var content = parser.parseFromString(xhttp.responseText, 'text/html');    }
-    catch(err) {
-        setStatus("Failed to fetch files list: " + err.message);
-        return;
-    }
-    
-    const list = content.querySelectorAll("a");
-    
-    var urls = [];
-    
-    for (a of list) {
-        url = a.getAttribute("href");
-        urls.push(getDomainname() + url);
-    }
-    
-    // Pack as zip and download
-    try {
-        backup(urls, zipFilename);
-        }
-    catch(err) {
-        setStatus("<span style=\"color: red\">Failed to zip files: " + err.message + "!</span>");
-        return;
-    }
-}
-
-
-function fetchFiles(urls, filesData, index, retry, zipFilename) {
-    url = urls[index];
-
-//    console.log(url + " started (" + index + "/" + urls.length + ")");
-    if (retry == 0) {
-        setStatus("&nbsp;- " + getFilenameFromUrl(urls[index]) + " (" + (index+1) + "/" + urls.length + ")...");
-    }
-    else {
-        setStatus("<span style=\"color: gray\">&nbsp;&nbsp;&nbsp;Retrying (" + retry + ")...</span>");
-    }
-
-    const xhr = new XMLHttpRequest();
-    xhr.open('GET', url, true);
-    xhr.responseType = "blob";
-
-    if (retry == 0) { // Short timeout on first retry
-        xhr.timeout = 2000; // time in milliseconds
-    }
-    else if (retry == 1) { // longer timeout
-        xhr.timeout = 5000; // time in milliseconds
-    }
-    else if (retry == 2) { // longer timeout
-        xhr.timeout = 10000; // time in milliseconds
-    }
-    else if (retry == 3) { // longer timeout
-        xhr.timeout = 20000; // time in milliseconds
-    }
-    else { // very long timeout
-        xhr.timeout = 30000; // time in milliseconds
-    }
-
-    xhr.onload = () => { // Request finished
-        //console.log(url + " done");
-
-        filesData[index] = xhr.response;
-
-        if (index == urls.length - 1) {
-            setStatus("Fetched all files");
-            generateZipFile(urls, filesData, zipFilename);
-            return;
-        }
-        else { // Next file
-            fetchFiles(urls, filesData, index+1, 0, zipFilename);
-        }
-    };
-
-    xhr.onprogress = (e) => { // XMLHttpRequest progress ... extend timeout
-        xhr.timeout = xhr.timeout + 500;
-    };
-
-    xhr.onerror = (e) => { // XMLHttpRequest error loading
-        console.log("Error on fetching " + url + "!");
-        if (retry > 5) {
-            setStatus("<span style=\"color: red\">Backup failed, please restart the device and try again!</span>");
-        }
-        else {
-            fetchFiles(urls, filesData, index, retry+1, zipFilename);
-        }
-    };
-
-    xhr.ontimeout = (e) => { // XMLHttpRequest timed out
-        console.log("Timeout on fetching " + url + "!");
-        if (retry > 5) {
-            setStatus("<span style=\"color: red\">Backup failed, please restart the device and try again!</span>");
-        }
-        else {
-            fetchFiles(urls, filesData, index, retry+1, zipFilename);
-        }
-    };
-
-    xhr.send(null);
-}
-
-
-function generateZipFile(urls, filesData, zipFilename) {
-    setStatus("Creating Zip File...");
-
-    var zip = new JSZip();
-
-    for (var i = 0; i < urls.length; i++) {        
-        zip.file(getFilenameFromUrl(urls[i]), filesData[i]);
-    }
-
-    zip.generateAsync({type:"blob"})
-    .then(function(content) {
-        saveAs(content, zipFilename);
-    });
-
-    setStatus("Backup completed");
-}
-
-
-const backup = (urls, zipFilename) => {
-    if(!urls) return;
-
-    /* Testing */
-    /*len = urls.length;
-    for (i = 0; i < len - 3; i++) {
-        urls.pop();
-    }*/
-
-    console.log(urls);
-
-    urlIndex = 0;
-    setStatus("Fetching files...");
-    fetchFiles(urls, [], 0, 0, zipFilename);
-};
-
-
-function setStatus(status) {
-    console.log(status);
-    document.getElementById("progress").innerHTML += status + "<br>\n";
-}
-
-function getFilenameFromUrl(url) {
-    return filename = url.substring(url.lastIndexOf('/')+1);
-}
-
-</script>
-
-</html>
+<!DOCTYPE html>
+<html lang="en" xml:lang="en"> 
+<head>
+    <title>Backup/Restore Configuration</title>
+    <meta charset="UTF-8" />
+
+    <style>
+        h1 {font-size: 2em;}
+        h2 {font-size: 1.5em; margin-block-start: 0.0em; margin-block-end: 0.2em;}
+        h3 {font-size: 1.2em;}
+        p {font-size: 1em;}
+
+        input[type=number] {
+            width: 138px;
+            padding: 10px 5px;
+            display: inline-block;
+            border: 1px solid #ccc;
+            font-size: 16px; 
+        }
+
+        .button {
+            padding: 5px 10px;
+            width: 205px;
+            font-size: 16px;
+        }
+    </style>
+
+</head>
+
+<body style="font-family: arial; padding: 0px 10px;">
+    <h2>Backup Configuration</h2>
+    <p>With the following action the <a href="/fileserver/config/" target="_self">config</a> folder on the SD-card gets zipped and provided as a download.</p>
+
+    <button class="button" id="startBackup" type="button" onclick="startBackup()">Create Backup</button>
+    <p id=progress></p>
+    <hr>
+    <h2>Restore Configuration</h2>
+    <p>Use the <a href="/fileserver/config/" target="_self">File Server</a> to upload individual files.</p>
+</body>
+
+
+<script type="text/javascript" src="common.js?v=$COMMIT_HASH"></script>
+<script type="text/javascript" src="jszip.min.js?v=$COMMIT_HASH"></script>
+<script type="text/javascript" src="FileSaver.min.js?v=$COMMIT_HASH"></script>
+<script>
+
+function startBackup() {  
+    document.getElementById("progress").innerHTML = "Creating backup...<br>\n";
+    
+    // Get hostname
+    try {
+        var xhttp = new XMLHttpRequest();
+        xhttp.open("GET", getDomainname() + "/info?type=Hostname", false);
+        xhttp.send();
+        hostname = xhttp.responseText;
+    }
+    catch(err) {
+        setStatus("<span style=\"color: red\">Failed to fetch hostname: " + err.message + "!</span>");
+        return;
+    }
+    
+    // get date/time
+    var dateTime = new Date().toJSON().slice(0,10) + "_" + new Date().toJSON().slice(11,19).replaceAll(":", "-");
+    
+    zipFilename = hostname + "_" + dateTime + ".zip";
+    console.log(zipFilename);
+
+    // Get files list
+    setStatus("Fetching File List...");
+    try {
+        var xhttp = new XMLHttpRequest();
+        xhttp.open("GET", getDomainname() + "/fileserver/config/", false);
+        xhttp.send();
+        
+        var parser = new DOMParser();
+        var content = parser.parseFromString(xhttp.responseText, 'text/html');    }
+    catch(err) {
+        setStatus("Failed to fetch files list: " + err.message);
+        return;
+    }
+    
+    const list = content.querySelectorAll("a");
+    
+    var urls = [];
+    
+    for (a of list) {
+        url = a.getAttribute("href");
+        urls.push(getDomainname() + url);
+    }
+    
+    // Pack as zip and download
+    try {
+        backup(urls, zipFilename);
+        }
+    catch(err) {
+        setStatus("<span style=\"color: red\">Failed to zip files: " + err.message + "!</span>");
+        return;
+    }
+}
+
+
+function fetchFiles(urls, filesData, index, retry, zipFilename) {
+    url = urls[index];
+
+//    console.log(url + " started (" + index + "/" + urls.length + ")");
+    if (retry == 0) {
+        setStatus("&nbsp;- " + getFilenameFromUrl(urls[index]) + " (" + (index+1) + "/" + urls.length + ")...");
+    }
+    else {
+        setStatus("<span style=\"color: gray\">&nbsp;&nbsp;&nbsp;Retrying (" + retry + ")...</span>");
+    }
+
+    const xhr = new XMLHttpRequest();
+    xhr.open('GET', url, true);
+    xhr.responseType = "blob";
+
+    if (retry == 0) { // Short timeout on first retry
+        xhr.timeout = 2000; // time in milliseconds
+    }
+    else if (retry == 1) { // longer timeout
+        xhr.timeout = 5000; // time in milliseconds
+    }
+    else if (retry == 2) { // longer timeout
+        xhr.timeout = 10000; // time in milliseconds
+    }
+    else if (retry == 3) { // longer timeout
+        xhr.timeout = 20000; // time in milliseconds
+    }
+    else { // very long timeout
+        xhr.timeout = 30000; // time in milliseconds
+    }
+
+    xhr.onload = () => { // Request finished
+        //console.log(url + " done");
+
+        filesData[index] = xhr.response;
+
+        if (index == urls.length - 1) {
+            setStatus("Fetched all files");
+            generateZipFile(urls, filesData, zipFilename);
+            return;
+        }
+        else { // Next file
+            fetchFiles(urls, filesData, index+1, 0, zipFilename);
+        }
+    };
+
+    xhr.onprogress = (e) => { // XMLHttpRequest progress ... extend timeout
+        xhr.timeout = xhr.timeout + 500;
+    };
+
+    xhr.onerror = (e) => { // XMLHttpRequest error loading
+        console.log("Error on fetching " + url + "!");
+        if (retry > 5) {
+            setStatus("<span style=\"color: red\">Backup failed, please restart the device and try again!</span>");
+        }
+        else {
+            fetchFiles(urls, filesData, index, retry+1, zipFilename);
+        }
+    };
+
+    xhr.ontimeout = (e) => { // XMLHttpRequest timed out
+        console.log("Timeout on fetching " + url + "!");
+        if (retry > 5) {
+            setStatus("<span style=\"color: red\">Backup failed, please restart the device and try again!</span>");
+        }
+        else {
+            fetchFiles(urls, filesData, index, retry+1, zipFilename);
+        }
+    };
+
+    xhr.send(null);
+}
+
+
+function generateZipFile(urls, filesData, zipFilename) {
+    setStatus("Creating Zip File...");
+
+    var zip = new JSZip();
+
+    for (var i = 0; i < urls.length; i++) {        
+        zip.file(getFilenameFromUrl(urls[i]), filesData[i]);
+    }
+
+    zip.generateAsync({type:"blob"})
+    .then(function(content) {
+        saveAs(content, zipFilename);
+    });
+
+    setStatus("Backup completed");
+}
+
+
+const backup = (urls, zipFilename) => {
+    if(!urls) return;
+
+    /* Testing */
+    /*len = urls.length;
+    for (i = 0; i < len - 3; i++) {
+        urls.pop();
+    }*/
+
+    console.log(urls);
+
+    urlIndex = 0;
+    setStatus("Fetching files...");
+    fetchFiles(urls, [], 0, 0, zipFilename);
+};
+
+
+function setStatus(status) {
+    console.log(status);
+    document.getElementById("progress").innerHTML += status + "<br>\n";
+}
+
+function getFilenameFromUrl(url) {
+    return filename = url.substring(url.lastIndexOf('/')+1);
+}
+
+</script>
+
+</html>
diff --git a/sd-card/html/data_export.html b/sd-card/html/data_export.html
new file mode 100644
index 00000000..786b868f
--- /dev/null
+++ b/sd-card/html/data_export.html
@@ -0,0 +1,207 @@
+<!DOCTYPE html>
+<html lang="en" xml:lang="en"> 
+<head>
+    <title>Data Export (CSV)</title>
+    <meta charset="UTF-8"/>
+
+    <style>
+        h1 {font-size: 2em;}
+        h2 {font-size: 1.5em; margin-block-start: 0.0em; margin-block-end: 0.2em;}
+        h3 {font-size: 1.2em;}
+        p {font-size: 1em;}
+
+        input[type=number] {
+            width: 138px;
+            padding: 10px 5px;
+            display: inline-block;
+            border: 1px solid #ccc;
+            font-size: 16px; 
+        }
+
+        .button {
+            padding: 5px 10px;
+            width: 300px;
+            font-size: 16px;
+        }
+    </style>
+</head>
+
+<body style="font-family: arial; padding: 0px 10px;">
+    <h2>Data Export(CSV)</h2>
+    <p>With the following action the <a href="/fileserver/log/data/" target="_self">data</a> folder on the SD-card gets zipped and provided as a download.</p>
+    <button class="button" id="startExportData" type="button" onclick="startExportData()">Export Data</button>
+    <p></p>
+    <hr>
+    <p id=progress></p>
+
+<script type="text/javascript" src="common.js?v=$COMMIT_HASH"></script>
+<script type="text/javascript" src="jszip.min.js?v=$COMMIT_HASH"></script>
+<script type="text/javascript" src="FileSaver.min.js?v=$COMMIT_HASH"></script>
+
+<script type="text/javascript">
+function startExportData() {  
+    document.getElementById("progress").innerHTML = "Creating Export Data...<br>\n";
+    
+    // Get hostname
+    try {
+        var xhttp = new XMLHttpRequest();
+        xhttp.open("GET", getDomainname() + "/info?type=Hostname", false);
+        xhttp.send();
+        hostname = xhttp.responseText;
+    }
+    catch(err) {
+        setStatus("<span style=\"color: red\">Failed to fetch hostname: " + err.message + "!</span>");
+        return;
+    }
+    
+    // get date/time
+    var dateTime = new Date().toJSON().slice(0,10) + "_" + new Date().toJSON().slice(11,19).replaceAll(":", "-");
+    
+    zipFilename = hostname + "_" + dateTime + "_csv" + ".zip";
+    console.log(zipFilename);
+
+    // Get files list
+    setStatus("Fetching File List...");
+    try {
+        var xhttp = new XMLHttpRequest();
+        xhttp.open("GET", getDomainname() + "/fileserver/log/data/", false);
+        xhttp.send();
+        
+        var parser = new DOMParser();
+        var content = parser.parseFromString(xhttp.responseText, 'text/html');
+    }
+    catch(err) {
+        setStatus("Failed to fetch files list: " + err.message);
+        return;
+    }
+    
+    const list = content.querySelectorAll("a");
+    
+    var urls = [];
+    
+    for (a of list) {
+        url = a.getAttribute("href");
+        urls.push(getDomainname() + url);
+    }
+    
+    // Pack as zip and download
+    try {
+        startExportZip(urls, zipFilename);
+        }
+    catch(err) {
+        setStatus("<span style=\"color: red\">Failed to zip files: " + err.message + "!</span>");
+        return;
+    }
+}
+
+function fetchFiles(urls, filesData, index, retry, zipFilename) {
+    url = urls[index];
+
+    if (retry == 0) {
+        setStatus("&nbsp;- " + getFilenameFromUrl(urls[index]) + " (" + (index+1) + "/" + urls.length + ")...");
+    }
+    else {
+        setStatus("<span style=\"color: gray\">&nbsp;&nbsp;&nbsp;Retrying (" + retry + ")...</span>");
+    }
+
+    const xhr = new XMLHttpRequest();
+    xhr.open('GET', url, true);
+    xhr.responseType = "blob";
+
+    if (retry == 0) { // Short timeout on first retry
+        xhr.timeout = 2000; // time in milliseconds
+    }
+    else if (retry == 1) { // longer timeout
+        xhr.timeout = 5000;
+    }
+    else if (retry == 2) { // longer timeout
+        xhr.timeout = 10000;
+    }
+    else if (retry == 3) { // longer timeout
+        xhr.timeout = 20000;
+    }
+    else { // very long timeout
+        xhr.timeout = 30000;
+    }
+
+    xhr.onload = () => { // Request finished
+        //console.log(url + " done");
+
+        filesData[index] = xhr.response;
+
+        if (index == urls.length - 1) {
+            setStatus("Fetched all files");
+            generateZipFile(urls, filesData, zipFilename);
+            return;
+        }
+        else { // Next file
+            fetchFiles(urls, filesData, index+1, 0, zipFilename);
+        }
+    };
+
+    xhr.onprogress = (e) => { // XMLHttpRequest progress ... extend timeout
+        xhr.timeout = xhr.timeout + 500;
+    };
+
+    xhr.onerror = (e) => { // XMLHttpRequest error loading
+        console.log("Error on fetching " + url + "!");
+        if (retry > 5) {
+            setStatus("<span style=\"color: red\">Data Export failed, please restart the device and try again!</span>");
+        }
+        else {
+            fetchFiles(urls, filesData, index, retry+1, zipFilename);
+        }
+    };
+
+    xhr.ontimeout = (e) => { // XMLHttpRequest timed out
+        console.log("Timeout on fetching " + url + "!");
+        if (retry > 5) {
+            setStatus("<span style=\"color: red\">Data Export failed, please restart the device and try again!</span>");
+        }
+        else {
+            fetchFiles(urls, filesData, index, retry+1, zipFilename);
+        }
+    };
+
+    xhr.send(null);
+}
+
+function generateZipFile(urls, filesData, zipFilename) {
+    setStatus("Creating Zip File...");
+
+    var zip = new JSZip();
+
+    for (var i = 0; i < urls.length; i++) {        
+        zip.file(getFilenameFromUrl(urls[i]), filesData[i]);
+    }
+
+    zip.generateAsync({type:"blob"})
+    .then(function(content) {
+        saveAs(content, zipFilename);
+    });
+
+    setStatus("Data Export completed");
+}
+
+const startExportZip = (urls, zipFilename) => {
+    if(!urls) { return; }
+
+    console.log(urls);
+
+    urlIndex = 0;
+    setStatus("Fetching files...");
+    fetchFiles(urls, [], 0, 0, zipFilename);
+};
+
+function setStatus(status) {
+    console.log(status);
+    document.getElementById("progress").innerHTML += status + "<br>\n";
+}
+
+function getFilenameFromUrl(url) {
+    return filename = url.substring(url.lastIndexOf('/')+1);
+}
+
+</script>
+</body>
+</html>
diff --git a/sd-card/html/index.html b/sd-card/html/index.html
index 0cb89bdc..0645ef05 100644
--- a/sd-card/html/index.html
+++ b/sd-card/html/index.html
@@ -142,6 +142,7 @@
             <li><a href="#" onclick="loadPage('graph.html?v=$COMMIT_HASH');">Data Graph</a></li>
             <li><a href="#" onclick="loadPage('data.html?v=$COMMIT_HASH');">Data Table</a></li>
             <li><a href="#" onclick="loadPage(getDomainname() + '/fileserver/log/data/');">Data Files</a></li>
+            <li><a href="#" onclick="loadPage('data_export.html?v=$COMMIT_HASH');">Data Export</a></li>
         </ul>
     </li>
 
