diff --git a/README.md b/README.md index 753b1741..7546ce24 100644 --- a/README.md +++ b/README.md @@ -19,18 +19,27 @@ A 3d-printable housing can be found here: https://www.thingiverse.com/thing:4571 ### Known Issues -* spontaneous reboot, especially in case of intensive web server access (improved since v2.1.0) +* reboot on extensive web access due to the limits of the internal web server ------ **General remark:** Beside the `firmware.bin`, typically also the content of `/html` needs to be updated! -##### Rolling - (2020-11-03) -* Bug-Fix in time sync on warm reboot -* based on v3.1.0 (2020-10-26) +##### Rolling - (2020-11-15) +* based on v4.0.0 (2020-11-15) + + + +##### 4.0.0 Tflite Core - (2020-11-15) +* Implementation of rolling log-files + +* Update Tflite-Core to master@20201108 (v2.4) + +* Bug-fixing for reducing reboots + ##### 3.1.0 MQTT-Client - (2020-10-26) @@ -52,7 +61,6 @@ A 3d-printable housing can be found here: https://www.thingiverse.com/thing:4571 * Bug-Fixing (hostname in wlan.ini and error handling inside flow) - ##### 2.2.0 Version Control (2020-09-27) @@ -61,8 +69,6 @@ A 3d-printable housing can be found here: https://www.thingiverse.com/thing:4571 * Update Build-System to PlatformIO - Espressif 32 v2.0.0 (ESP-IDF 4.1) - - ##### 2.1.0 Decimal Shift, Chrome & Edge (2020-09-25) * Implementation of Decimal Shift @@ -79,7 +85,6 @@ A 3d-printable housing can be found here: https://www.thingiverse.com/thing:4571 * Bug fixing, code corrections - ##### 2.0.0 Layout update (2020-09-12) diff --git a/code/lib/jomjol_fileserver_ota/server_file.cpp b/code/lib/jomjol_fileserver_ota/server_file.cpp index 633542bc..78f45a74 100644 --- a/code/lib/jomjol_fileserver_ota/server_file.cpp +++ b/code/lib/jomjol_fileserver_ota/server_file.cpp @@ -84,7 +84,7 @@ static esp_err_t favicon_get_handler(httpd_req_t *req) * a list of all files and folders under the requested path. * In case of SPIFFS this returns empty list when path is any * string other than '/', since SPIFFS doesn't support directories */ -static esp_err_t http_resp_dir_html(httpd_req_t *req, const char *dirpath) +static esp_err_t http_resp_dir_html(httpd_req_t *req, const char *dirpath, const char* uripath, bool readonly) { char entrypath[FILE_PATH_MAX]; char entrysize[16]; @@ -120,24 +120,24 @@ static esp_err_t http_resp_dir_html(httpd_req_t *req, const char *dirpath) httpd_resp_sendstr_chunk(req, ""); ///////////////////////////////////////////////// - - FILE *fd = fopen("/sdcard/html/upload_script.html", "r"); - char *chunk = ((struct file_server_data *)req->user_ctx)->scratch; - size_t chunksize; - do { - chunksize = fread(chunk, 1, SCRATCH_BUFSIZE, fd); -// printf("Chunksize %d\n", chunksize); - if (chunksize > 0){ - if (httpd_resp_send_chunk(req, chunk, chunksize) != ESP_OK) { - fclose(fd); - ESP_LOGE(TAG, "File sending failed!"); - return ESP_FAIL; + if (!readonly) { + FILE *fd = fopen("/sdcard/html/upload_script.html", "r"); + char *chunk = ((struct file_server_data *)req->user_ctx)->scratch; + size_t chunksize; + do { + chunksize = fread(chunk, 1, SCRATCH_BUFSIZE, fd); + // printf("Chunksize %d\n", chunksize); + if (chunksize > 0){ + if (httpd_resp_send_chunk(req, chunk, chunksize) != ESP_OK) { + fclose(fd); + ESP_LOGE(TAG, "File sending failed!"); + return ESP_FAIL; + } } - } - } while (chunksize != 0); - fclose(fd); -// ESP_LOGI(TAG, "File sending complete"); - + } while (chunksize != 0); + fclose(fd); + // ESP_LOGI(TAG, "File sending complete"); + } /////////////////////////////// std::string _zw = std::string(dirpath); @@ -149,12 +149,16 @@ static esp_err_t http_resp_dir_html(httpd_req_t *req, const char *dirpath) httpd_resp_sendstr_chunk(req, "" "" - "\n"); + ""); + if (!readonly) { + httpd_resp_sendstr_chunk(req, ""); + } + httpd_resp_sendstr_chunk(req, "\n"); /* Iterate over all files / folders and fetch their names and sizes */ while ((entry = readdir(dir)) != NULL) { @@ -173,7 +177,8 @@ static esp_err_t http_resp_dir_html(httpd_req_t *req, const char *dirpath) /* Send chunk of HTML file containing table entries with file name and size */ httpd_resp_sendstr_chunk(req, "\n"); } } @@ -226,6 +233,7 @@ static esp_err_t download_get_handler(httpd_req_t *req) // filename = get_path_from_uri(filepath, ((struct file_server_data *)req->user_ctx)->base_path, // req->uri, sizeof(filepath)); + if (!filename) { ESP_LOGE(TAG, "Filename is too long"); /* Respond with 500 Internal Server Error */ @@ -235,7 +243,22 @@ static esp_err_t download_get_handler(httpd_req_t *req) /* If name has trailing '/', respond with directory contents */ if (filename[strlen(filename) - 1] == '/') { - return http_resp_dir_html(req, filepath); + bool readonly = false; + size_t buf_len = httpd_req_get_url_query_len(req) + 1; + if (buf_len > 1) { + char buf[buf_len]; + if (httpd_req_get_url_query_str(req, buf, buf_len) == ESP_OK) { + ESP_LOGI(TAG, "Found URL query => %s", buf); + char param[32]; + /* Get value of expected key from query string */ + if (httpd_query_key_value(buf, "readonly", param, sizeof(param)) == ESP_OK) { + ESP_LOGI(TAG, "Found URL query parameter => readonly=%s", param); + readonly = param && strcmp(param,"true")==0; + } + } + } + + return http_resp_dir_html(req, filepath, filename, readonly); } std::string testwlan = toUpper(std::string(filename)); diff --git a/code/lib/jomjol_flowcontroll/ClassFlow.h b/code/lib/jomjol_flowcontroll/ClassFlow.h index 35928d91..18260229 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlow.h +++ b/code/lib/jomjol_flowcontroll/ClassFlow.h @@ -9,6 +9,10 @@ using namespace std; +#define LOGFILE_TIME_FORMAT "%Y%m%d-%H%M%S" +#define LOGFILE_TIME_FORMAT_DATE_EXTR substr(0, 8) +#define LOGFILE_TIME_FORMAT_HOUR_EXTR substr(9, 2) + struct HTMLInfo { float val; diff --git a/code/lib/jomjol_flowcontroll/ClassFlowAnalog.cpp b/code/lib/jomjol_flowcontroll/ClassFlowAnalog.cpp index fa6085e7..d0dea860 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowAnalog.cpp +++ b/code/lib/jomjol_flowcontroll/ClassFlowAnalog.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include // #define OHNETFLITE @@ -12,25 +12,23 @@ #include "ClassLogFile.h" +static const char* TAG = "flow_analog"; + bool debugdetailanalog = false; -ClassFlowAnalog::ClassFlowAnalog() +ClassFlowAnalog::ClassFlowAnalog() : ClassFlowImage(TAG) { - isLogImage = false; string cnnmodelfile = ""; modelxsize = 1; modelysize = 1; ListFlowControll = NULL; } -ClassFlowAnalog::ClassFlowAnalog(std::vector* lfc) +ClassFlowAnalog::ClassFlowAnalog(std::vector* lfc) : ClassFlowImage(lfc, TAG) { - isLogImage = false; string cnnmodelfile = ""; modelxsize = 1; modelysize = 1; - ListFlowControll = NULL; - ListFlowControll = lfc; } @@ -90,8 +88,12 @@ bool ClassFlowAnalog::ReadParameter(FILE* pfile, string& aktparamgraph) zerlegt = this->ZerlegeZeile(aktparamgraph); if ((zerlegt[0] == "LogImageLocation") && (zerlegt.size() > 1)) { + this->LogImageLocation = "/sdcard" + zerlegt[1]; this->isLogImage = true; - this->LogImageLocation = zerlegt[1]; + } + if ((toUpper(zerlegt[0]) == "LOGFILERETENTIONINDAYS") && (zerlegt.size() > 1)) + { + this->logfileRetentionInDays = std::stoi(zerlegt[1]); } if ((zerlegt[0] == "Model") && (zerlegt.size() > 1)) { @@ -153,6 +155,8 @@ bool ClassFlowAnalog::doFlow(string time) doNeuralNetwork(time); + RemoveOldLogs(); + return true; } @@ -233,10 +237,11 @@ bool ClassFlowAnalog::doAlignAndCut(string time) bool ClassFlowAnalog::doNeuralNetwork(string time) { + string logPath = CreateLogFolder(time); + string input = "/sdcard/img_tmp/alg.jpg"; string ioresize = "/sdcard/img_tmp/resize.bmp"; string output; - string nm; input = FormatFileName(input); #ifndef OHNETFLITE @@ -275,19 +280,7 @@ bool ClassFlowAnalog::doNeuralNetwork(string time) printf("Result Analog%i: %f\n", i, ROI[i]->result); - if (isLogImage) - { - std::stringstream stream; - stream << std::fixed << std::setprecision(1) << ROI[i]->result; - std::string s = stream.str(); -// std::snprintf(&s[0], s.size(), "%.2f", pi); - nm = "/sdcard" + LogImageLocation + "/" + s + "_" + ROI[i]->name + "_" + time + ".jpg"; - nm = FormatFileName(nm); - output = "/sdcard/img_tmp/" + ROI[i]->name + ".jpg"; - output = FormatFileName(output); - printf("Analog - save to file: %s\n", nm.c_str()); - CopyFile(output, nm); - } + LogImage(logPath, ROI[i]->name, &ROI[i]->result, NULL, time); } #ifndef OHNETFLITE delete tflite; diff --git a/code/lib/jomjol_flowcontroll/ClassFlowAnalog.h b/code/lib/jomjol_flowcontroll/ClassFlowAnalog.h index ce41f17f..89f2f6e2 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowAnalog.h +++ b/code/lib/jomjol_flowcontroll/ClassFlowAnalog.h @@ -1,5 +1,5 @@ #pragma once -#include "ClassFlow.h" +#include "ClassFlowImage.h" // #include "CTfLiteClass.h" struct roianalog { @@ -10,11 +10,9 @@ struct roianalog { class ClassFlowAnalog : - public ClassFlow + public ClassFlowImage { protected: - string LogImageLocation; - bool isLogImage; std::vector ROI; string cnnmodelfile; int modelxsize, modelysize; diff --git a/code/lib/jomjol_flowcontroll/ClassFlowControll.cpp b/code/lib/jomjol_flowcontroll/ClassFlowControll.cpp index 50ea08c7..a50de987 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowControll.cpp +++ b/code/lib/jomjol_flowcontroll/ClassFlowControll.cpp @@ -1,10 +1,14 @@ #include "ClassFlowControll.h" +#include +#include #include "ClassLogFile.h" #include "time_sntp.h" #include "Helper.h" #include "server_ota.h" +static const char* TAG = "flow_controll"; + std::string ClassFlowControll::doSingleStep(std::string _stepname, std::string _host){ std::string _classname = ""; std::string result = ""; @@ -149,6 +153,8 @@ std::string ClassFlowControll::getActStatus(){ bool ClassFlowControll::doFlow(string time) { +// CleanTempFolder(); // dazu muss man noch eine Rolling einführen + bool result = true; std::string zw_time; int repeat = 0; @@ -280,8 +286,41 @@ bool ClassFlowControll::ReadParameter(FILE* pfile, string& aktparamgraph) { LogFile.SwitchOnOff(false); } + } + if ((toUpper(zerlegt[0]) == "LOGFILERETENTIONINDAYS") && (zerlegt.size() > 1)) + { + LogFile.SetRetention(std::stoi(zerlegt[1])); } } return true; } +int ClassFlowControll::CleanTempFolder() { + const char* folderPath = "/sdcard/img_tmp"; + + ESP_LOGI(TAG, "Clean up temporary folder to avoid damage of sdcard sectors : %s", folderPath); + DIR *dir = opendir(folderPath); + if (!dir) { + ESP_LOGE(TAG, "Failed to stat dir : %s", folderPath); + return -1; + } + + struct dirent *entry; + int deleted = 0; + while ((entry = readdir(dir)) != NULL) { + std::string path = string(folderPath) + "/" + entry->d_name; + if (entry->d_type == DT_REG) { + if (unlink(path.c_str()) == 0) { + deleted ++; + } else { + ESP_LOGE(TAG, "can't delete file : %s", path.c_str()); + } + } else if (entry->d_type == DT_DIR) { + deleted += removeFolder(path.c_str(), TAG); + } + } + closedir(dir); + ESP_LOGI(TAG, "%d files deleted", deleted); + + return 0; +} \ No newline at end of file diff --git a/code/lib/jomjol_flowcontroll/ClassFlowControll.h b/code/lib/jomjol_flowcontroll/ClassFlowControll.h index d3d9de9a..91eac5fa 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowControll.h +++ b/code/lib/jomjol_flowcontroll/ClassFlowControll.h @@ -42,6 +42,8 @@ public: std::vector GetAllDigital(); std::vector GetAllAnalog(); + int CleanTempFolder(); + string name(){return "ClassFlowControll";}; }; diff --git a/code/lib/jomjol_flowcontroll/ClassFlowDigit.cpp b/code/lib/jomjol_flowcontroll/ClassFlowDigit.cpp index 70f1132b..90f02254 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowDigit.cpp +++ b/code/lib/jomjol_flowcontroll/ClassFlowDigit.cpp @@ -13,23 +13,21 @@ #include "ClassLogFile.h" -ClassFlowDigit::ClassFlowDigit() +static const char* TAG = "flow_digital"; + +ClassFlowDigit::ClassFlowDigit() : ClassFlowImage(TAG) { - isLogImage = false; string cnnmodelfile = ""; modelxsize = 1; modelysize = 1; ListFlowControll = NULL; } -ClassFlowDigit::ClassFlowDigit(std::vector* lfc) +ClassFlowDigit::ClassFlowDigit(std::vector* lfc) : ClassFlowImage(lfc, TAG) { - isLogImage = false; string cnnmodelfile = ""; modelxsize = 1; modelysize = 1; - ListFlowControll = NULL; - ListFlowControll = lfc; } string ClassFlowDigit::getReadout() @@ -66,8 +64,8 @@ bool ClassFlowDigit::ReadParameter(FILE* pfile, string& aktparamgraph) zerlegt = this->ZerlegeZeile(aktparamgraph); if ((zerlegt[0] == "LogImageLocation") && (zerlegt.size() > 1)) { - isLogImage = true; - LogImageLocation = zerlegt[1]; + LogImageLocation = "/sdcard" + zerlegt[1]; + isLogImage = true; } if ((zerlegt[0] == "Model") && (zerlegt.size() > 1)) { @@ -128,6 +126,8 @@ bool ClassFlowDigit::doFlow(string time) doNeuralNetwork(time); + RemoveOldLogs(); + return true; } @@ -194,6 +194,8 @@ bool ClassFlowDigit::doAlignAndCut(string time) bool ClassFlowDigit::doNeuralNetwork(string time) { + string logPath = CreateLogFolder(time); + string input = "/sdcard/img_tmp/alg.jpg"; string ioresize = "/sdcard/img_tmp/resize.bmp"; string output; @@ -221,16 +223,9 @@ bool ClassFlowDigit::doNeuralNetwork(string time) #ifndef OHNETFLITE ROI[i]->resultklasse = tflite->GetClassFromImage(ioresize); #endif - printf("Result Digit%i: %d\n", i, ROI[i]->resultklasse); + printf("Result Digit%i: %d\n", i, ROI[i]->resultklasse); - if (isLogImage) - { - nm = "/sdcard" + LogImageLocation + "/" + std::to_string(ROI[i]->resultklasse) + "/" + time + "_" + ROI[i]->name + ".jpg"; - output = "/sdcard/img_tmp/" + ROI[i]->name + ".jpg"; - output = FormatFileName(output); - nm = FormatFileName(nm); - CopyFile(output, nm); - } + LogImage(logPath, ROI[i]->name, NULL, &ROI[i]->resultklasse, time); } #ifndef OHNETFLITE delete tflite; diff --git a/code/lib/jomjol_flowcontroll/ClassFlowDigit.h b/code/lib/jomjol_flowcontroll/ClassFlowDigit.h index b84c7f37..3546bb36 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowDigit.h +++ b/code/lib/jomjol_flowcontroll/ClassFlowDigit.h @@ -1,5 +1,5 @@ #pragma once -#include "ClassFlow.h" +#include "ClassFlowImage.h" #include "Helper.h" #include @@ -12,11 +12,9 @@ struct roi { }; class ClassFlowDigit : - public ClassFlow + public ClassFlowImage { protected: - string LogImageLocation; - bool isLogImage; std::vector ROI; string cnnmodelfile; int modelxsize, modelysize; diff --git a/code/lib/jomjol_flowcontroll/ClassFlowImage.cpp b/code/lib/jomjol_flowcontroll/ClassFlowImage.cpp new file mode 100644 index 00000000..5c09160e --- /dev/null +++ b/code/lib/jomjol_flowcontroll/ClassFlowImage.cpp @@ -0,0 +1,101 @@ +#include "ClassFlowImage.h" +#include +#include +#include +#include +#include "time_sntp.h" +#include "ClassLogFile.h" + +ClassFlowImage::ClassFlowImage(const char* logTag) +{ + this->logTag = logTag; + isLogImage = false; +} + +ClassFlowImage::ClassFlowImage(std::vector * lfc, const char* logTag) : ClassFlow((std::vector*)lfc) +{ + this->logTag = logTag; + isLogImage = false; +} + +string ClassFlowImage::CreateLogFolder(string time) { + if (!isLogImage) + return ""; + + string logPath = LogImageLocation + "/" + time.LOGFILE_TIME_FORMAT_DATE_EXTR + "/" + time.LOGFILE_TIME_FORMAT_HOUR_EXTR; + isLogImage = mkdir_r(logPath.c_str(), S_IRWXU) == 0; + if (!isLogImage) { + ESP_LOGW(logTag, "Can't create log foolder for analog images. Path %s", logPath.c_str()); + LogFile.WriteToFile("Can't create log foolder for analog images. Path " + logPath); + } + + return logPath; +} + +void ClassFlowImage::LogImage(string logPath, string name, float *resultFloat, int *resultInt, string time) { + if (!isLogImage) + return; + + char buf[10]; + if (resultFloat != NULL) { + sprintf(buf, "%.1f_", *resultFloat); + } else if (resultInt != NULL) { + sprintf(buf, "%d_", *resultInt); + } else { + buf[0] = '\0'; + } + + string nm = logPath + "/" + buf + name + "_" + time + ".jpg"; + nm = FormatFileName(nm); + string output = "/sdcard/img_tmp/" + name + ".jpg"; + output = FormatFileName(output); + printf("save to file: %s\n", nm.c_str()); + CopyFile(output, nm); +} + +void ClassFlowImage::RemoveOldLogs() +{ + if (!isLogImage) + return; + + ESP_LOGI(logTag, "remove old log images"); + if (logfileRetentionInDays == 0) { + return; + } + + time_t rawtime; + struct tm* timeinfo; + char cmpfilename[30]; + + time(&rawtime); + rawtime = addDays(rawtime, -logfileRetentionInDays); + timeinfo = localtime(&rawtime); + + strftime(cmpfilename, 30, LOGFILE_TIME_FORMAT, timeinfo); + //ESP_LOGE(TAG, "log file name to compare: %s", cmpfilename); + string folderName = string(cmpfilename).LOGFILE_TIME_FORMAT_DATE_EXTR; + + DIR *dir = opendir(LogImageLocation.c_str()); + if (!dir) { + ESP_LOGI(logTag, "Failed to stat dir : %s", LogImageLocation.c_str()); + return; + } + + struct dirent *entry; + int deleted = 0; + int notDeleted = 0; + while ((entry = readdir(dir)) != NULL) { + string folderPath = LogImageLocation + "/" + entry->d_name; + if (entry->d_type == DT_DIR) { + //ESP_LOGI(logTag, "Compare %s %s", entry->d_name, folderName.c_str()); + if ((strlen(entry->d_name) == folderName.length()) && (strcmp(entry->d_name, folderName.c_str()) < 0)) { + deleted += removeFolder(folderPath.c_str(), logTag); + } else { + notDeleted ++; + } + } + } + ESP_LOGI(logTag, "%d older log files deleted. %d current log files not deleted.", deleted, notDeleted); + closedir(dir); +} + diff --git a/code/lib/jomjol_flowcontroll/ClassFlowImage.h b/code/lib/jomjol_flowcontroll/ClassFlowImage.h new file mode 100644 index 00000000..23154c64 --- /dev/null +++ b/code/lib/jomjol_flowcontroll/ClassFlowImage.h @@ -0,0 +1,22 @@ +#pragma once +#include "ClassFlow.h" + +using namespace std; + +class ClassFlowImage : public ClassFlow +{ +protected: + string LogImageLocation; + bool isLogImage; + unsigned short logfileRetentionInDays; + const char* logTag; + + string CreateLogFolder(string time); + void LogImage(string logPath, string name, float *resultFloat, int *resultInt, string time); + +public: + ClassFlowImage(const char* logTag); + ClassFlowImage(std::vector * lfc, const char* logTag); + + void RemoveOldLogs(); +}; diff --git a/code/lib/jomjol_flowcontroll/ClassFlowMQTT.cpp b/code/lib/jomjol_flowcontroll/ClassFlowMQTT.cpp index c1d66426..92396e2b 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowMQTT.cpp +++ b/code/lib/jomjol_flowcontroll/ClassFlowMQTT.cpp @@ -6,8 +6,6 @@ #include -static const char* TAG2 = "example"; - ClassFlowMQTT::ClassFlowMQTT() { uri = ""; diff --git a/code/lib/jomjol_flowcontroll/ClassFlowMakeImage.cpp b/code/lib/jomjol_flowcontroll/ClassFlowMakeImage.cpp index d7f3d201..d1f4d78c 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowMakeImage.cpp +++ b/code/lib/jomjol_flowcontroll/ClassFlowMakeImage.cpp @@ -1,12 +1,12 @@ #include "ClassFlowMakeImage.h" #include "Helper.h" - #include "CFindTemplate.h" #include "ClassControllCamera.h" #include +static const char* TAG = "flow_make_image"; esp_err_t ClassFlowMakeImage::camera_capture(){ string nm = namerawimage; @@ -24,11 +24,8 @@ void ClassFlowMakeImage::takePictureWithFlash(int flashdauer) } - - -ClassFlowMakeImage::ClassFlowMakeImage() +ClassFlowMakeImage::ClassFlowMakeImage() : ClassFlowImage(TAG) { - isLogImage = false; waitbeforepicture = 5; isImageSize = false; ImageQuality = -1; @@ -36,16 +33,13 @@ ClassFlowMakeImage::ClassFlowMakeImage() namerawimage = "/sdcard/img_tmp/raw.jpg"; } -ClassFlowMakeImage::ClassFlowMakeImage(std::vector* lfc) +ClassFlowMakeImage::ClassFlowMakeImage(std::vector* lfc) : ClassFlowImage(lfc, TAG) { - isLogImage = false; waitbeforepicture = 5; isImageSize = false; ImageQuality = -1; TimeImageTaken = 0; namerawimage = "/sdcard/img_tmp/raw.jpg"; - - ListFlowControll = lfc; } bool ClassFlowMakeImage::ReadParameter(FILE* pfile, string& aktparamgraph) @@ -66,8 +60,8 @@ bool ClassFlowMakeImage::ReadParameter(FILE* pfile, string& aktparamgraph) zerlegt = this->ZerlegeZeile(aktparamgraph); if ((zerlegt[0] == "LogImageLocation") && (zerlegt.size() > 1)) { - this->isLogImage = true; - this->LogImageLocation = zerlegt[1]; + LogImageLocation = "/sdcard" + zerlegt[1]; + isLogImage = true; } if ((zerlegt[0] == "ImageQuality") && (zerlegt.size() > 1)) this->ImageQuality = std::stod(zerlegt[1]); @@ -81,45 +75,6 @@ bool ClassFlowMakeImage::ReadParameter(FILE* pfile, string& aktparamgraph) return true; } - -void ClassFlowMakeImage::CopyFile(string input, string output) -{ - input = FormatFileName(input); - output = FormatFileName(output); - input = namerawimage; - - - printf("Copy Input : %s\n", input.c_str()); - printf("Copy Output: %s\n", output.c_str()); - - char cTemp; - FILE* fpSourceFile = fopen(input.c_str(), "rb"); - FILE* fpTargetFile = fopen(output.c_str(), "wb"); - - if (fpSourceFile == NULL) - { - printf("fpSourceFile == NULL\n"); - perror("Error"); - } - - if (fpTargetFile == NULL) - { - printf("fpTargetFile == NULL\n"); - perror("Error"); - } - - - while (fread(&cTemp, 1, 1, fpSourceFile) == 1) - { - fwrite(&cTemp, 1, 1, fpTargetFile); - } - - // Close The Files - fclose(fpSourceFile); - fclose(fpTargetFile); - printf("Copy done\n"); -} - string ClassFlowMakeImage::getHTMLSingleStep(string host) { string result; @@ -133,6 +88,8 @@ bool ClassFlowMakeImage::doFlow(string zwtime) // TakeImage and Store into /image_tmp/raw.jpg TO BE DONE //////////////////////////////////////////////////////////////////// + string logPath = CreateLogFolder(zwtime); + int flashdauer = (int) waitbeforepicture * 1000; @@ -140,16 +97,9 @@ bool ClassFlowMakeImage::doFlow(string zwtime) time(&TimeImageTaken); localtime(&TimeImageTaken); + LogImage(logPath, "raw", NULL, NULL, zwtime); - if (this->isLogImage) - { - string nm = "/sdcard" + this->LogImageLocation + "/" + zwtime + ".jpg"; - string input = "/sdcard/image_tmp/raw.jgp"; - printf("loginput from: %s to: %s\n", input.c_str(), nm.c_str()); - nm = FormatFileName(nm); - input = FormatFileName(input); - CopyFile(input, nm); - } + RemoveOldLogs(); return true; } diff --git a/code/lib/jomjol_flowcontroll/ClassFlowMakeImage.h b/code/lib/jomjol_flowcontroll/ClassFlowMakeImage.h index f19eef94..febb2ab3 100644 --- a/code/lib/jomjol_flowcontroll/ClassFlowMakeImage.h +++ b/code/lib/jomjol_flowcontroll/ClassFlowMakeImage.h @@ -1,11 +1,9 @@ #pragma once -#include "ClassFlow.h" +#include "ClassFlowImage.h" #include "ClassControllCamera.h" #include -static const char* TAG2 = "example"; - #define BLINK_GPIO GPIO_NUM_4 #define CAMERA_MODEL_AI_THINKER @@ -13,11 +11,9 @@ static const char* TAG2 = "example"; class ClassFlowMakeImage : - public ClassFlow + public ClassFlowImage { protected: - string LogImageLocation; - bool isLogImage; float waitbeforepicture; framesize_t ImageSize; bool isImageSize; diff --git a/code/lib/jomjol_helper/Helper.cpp b/code/lib/jomjol_helper/Helper.cpp index 4e7797ec..5ffd73e0 100644 --- a/code/lib/jomjol_helper/Helper.cpp +++ b/code/lib/jomjol_helper/Helper.cpp @@ -1,8 +1,14 @@ //#pragma warning(disable : 4996) #include "Helper.h" +#include +#include +#include +#include +#include //#define ISWINDOWS_TRUE +#define PATH_MAX_STRING_SIZE 256 using namespace std; @@ -159,6 +165,63 @@ string getFileType(string filename) return zw; } +/* recursive mkdir */ +int mkdir_r(const char *dir, const mode_t mode) { + char tmp[PATH_MAX_STRING_SIZE]; + char *p = NULL; + struct stat sb; + size_t len; + + /* copy path */ + len = strnlen (dir, PATH_MAX_STRING_SIZE); + if (len == 0 || len == PATH_MAX_STRING_SIZE) { + return -1; + } + memcpy (tmp, dir, len); + tmp[len] = '\0'; + + /* remove trailing slash */ + if(tmp[len - 1] == '/') { + tmp[len - 1] = '\0'; + } + + /* check if path exists and is a directory */ + if (stat (tmp, &sb) == 0) { + if (S_ISDIR (sb.st_mode)) { + return 0; + } + } + + /* recursive mkdir */ + for(p = tmp + 1; *p; p++) { + if(*p == '/') { + *p = 0; + /* test path */ + if (stat(tmp, &sb) != 0) { + /* path does not exist - create directory */ + if (mkdir(tmp, mode) < 0) { + return -1; + } + } else if (!S_ISDIR(sb.st_mode)) { + /* not a directory */ + return -1; + } + *p = '/'; + } + } + /* test path */ + if (stat(tmp, &sb) != 0) { + /* path does not exist - create directory */ + if (mkdir(tmp, mode) < 0) { + return -1; + } + } else if (!S_ISDIR(sb.st_mode)) { + /* not a directory */ + return -1; + } + return 0; +} + string toUpper(string in) { for (int i = 0; i < in.length(); ++i) @@ -173,3 +236,42 @@ float temperatureRead() { return (temprature_sens_read() - 32) / 1.8; } + +time_t addDays(time_t startTime, int days) { + struct tm* tm = localtime(&startTime); + tm->tm_mday += days; + return mktime(tm); +} + +int removeFolder(const char* folderPath, const char* logTag) { + ESP_LOGI(logTag, "Delete folder %s", folderPath); + + DIR *dir = opendir(folderPath); + if (!dir) { + ESP_LOGI(logTag, "Failed to stat dir : %s", folderPath); + return -1; + } + + struct dirent *entry; + int deleted = 0; + while ((entry = readdir(dir)) != NULL) { + std::string path = string(folderPath) + "/" + entry->d_name; + if (entry->d_type == DT_REG) { + if (unlink(path.c_str()) == 0) { + deleted ++; + } else { + ESP_LOGE(logTag, "can't delete file : %s", path.c_str()); + } + } else if (entry->d_type == DT_DIR) { + deleted += removeFolder(path.c_str(), logTag); + } + } + + closedir(dir); + if (rmdir(folderPath) != 0) { + ESP_LOGE(logTag, "can't delete file : %s", folderPath); + } + ESP_LOGI(logTag, "%d older log files in folder %s deleted.", deleted, folderPath); + + return deleted; +} diff --git a/code/lib/jomjol_helper/Helper.h b/code/lib/jomjol_helper/Helper.h index ce2f4e32..7891263c 100644 --- a/code/lib/jomjol_helper/Helper.h +++ b/code/lib/jomjol_helper/Helper.h @@ -17,6 +17,11 @@ bool ctype_space(const char c, string adddelimiter); string getFileType(string filename); +int mkdir_r(const char *dir, const mode_t mode); +int removeFolder(const char* folderPath, const char* logTag); + string toUpper(string in); float temperatureRead(); + +time_t addDays(time_t startTime, int days); diff --git a/code/lib/jomjol_logfile/ClassLogFile.cpp b/code/lib/jomjol_logfile/ClassLogFile.cpp index 11a168a6..ccb75cf0 100644 --- a/code/lib/jomjol_logfile/ClassLogFile.cpp +++ b/code/lib/jomjol_logfile/ClassLogFile.cpp @@ -1,7 +1,14 @@ #include "ClassLogFile.h" #include "time_sntp.h" +#include +#include +#include +#include +#include "Helper.h" -ClassLogFile LogFile("/sdcard/log.txt"); +static const char *TAG = "log"; + +ClassLogFile LogFile("/sdcard/log/message", "log_%Y-%m-%d.txt"); void ClassLogFile::WriteToDedicatedFile(std::string _fn, std::string info, bool _time) { @@ -13,39 +20,111 @@ void ClassLogFile::WriteToDedicatedFile(std::string _fn, std::string info, bool } pFile = fopen(_fn.c_str(), "a+"); + if (pFile!=NULL) { + if (_time) + { + time_t rawtime; + struct tm* timeinfo; + char buffer[80]; - if (_time) - { - time_t rawtime; - struct tm* timeinfo; - char buffer[80]; + time(&rawtime); + timeinfo = localtime(&rawtime); - time(&rawtime); - timeinfo = localtime(&rawtime); + strftime(buffer, 80, "%Y-%m-%d_%H-%M-%S", timeinfo); - strftime(buffer, 80, "%Y-%m-%d_%H-%M-%S", timeinfo); + zwtime = std::string(buffer); + info = zwtime + ": " + info; + } + fputs(info.c_str(), pFile); + fputs("\n", pFile); - zwtime = std::string(buffer); - info = zwtime + ": " + info; + fclose(pFile); + } else { + ESP_LOGI(TAG, "Can't open log file %s", _fn.c_str()); } - fputs(info.c_str(), pFile); - fputs("\n", pFile); - - fclose(pFile); } void ClassLogFile::SwitchOnOff(bool _doLogFile){ doLogFile = _doLogFile; }; +void ClassLogFile::SetRetention(unsigned short _retentionInDays){ + retentionInDays = _retentionInDays; +}; void ClassLogFile::WriteToFile(std::string info, bool _time) { - WriteToDedicatedFile(logfile, info, _time); + struct stat path_stat; + if (stat(logroot.c_str(), &path_stat) != 0) { + ESP_LOGI(TAG, "Create log folder: %s", logroot.c_str()); + if (mkdir_r(logroot.c_str(), S_IRWXU) == -1) { + ESP_LOGI(TAG, "Can't create log foolder"); + } + } + + time_t rawtime; + struct tm* timeinfo; + char buffer[30]; + + time(&rawtime); + timeinfo = localtime(&rawtime); + + strftime(buffer, 30, logfile.c_str(), timeinfo); + std::string logpath = logroot + "/" + buffer; + + WriteToDedicatedFile(logpath, info, _time); } -ClassLogFile::ClassLogFile(std::string _logfile) +void ClassLogFile::RemoveOld() { - logfile = _logfile; + if (retentionInDays == 0) { + return; + } + + time_t rawtime; + struct tm* timeinfo; + char cmpfilename[30]; + + time(&rawtime); + rawtime = addDays(rawtime, -retentionInDays); + timeinfo = localtime(&rawtime); + + strftime(cmpfilename, 30, logfile.c_str(), timeinfo); + //ESP_LOGE(TAG, "log file name to compare: %s", cmpfilename); + + DIR *dir = opendir(logroot.c_str()); + if (!dir) { + ESP_LOGI(TAG, "Failed to stat dir : %s", logroot.c_str()); + return; + } + + struct dirent *entry; + int deleted = 0; + int notDeleted = 0; + while ((entry = readdir(dir)) != NULL) { + if (entry->d_type == DT_REG) { + //ESP_LOGI(TAG, "list log file : %s %s", entry->d_name, cmpfilename); + if ((strlen(entry->d_name) == strlen(cmpfilename)) && (strcmp(entry->d_name, cmpfilename) < 0)) { + ESP_LOGI(TAG, "delete log file : %s", entry->d_name); + std::string filepath = logroot + "/" + entry->d_name; + if (unlink(filepath.c_str()) == 0) { + deleted ++; + } else { + ESP_LOGE(TAG, "can't delete file : %s", entry->d_name); + } + } else { + notDeleted ++; + } + } + } + ESP_LOGI(TAG, "%d older log files deleted. %d current log files not deleted.", deleted, notDeleted); + closedir(dir); +} + +ClassLogFile::ClassLogFile(std::string _logroot, std::string _logfile) +{ + logroot = _logroot; + logfile = _logfile; doLogFile = true; -} \ No newline at end of file + retentionInDays = 10; +} diff --git a/code/lib/jomjol_logfile/ClassLogFile.h b/code/lib/jomjol_logfile/ClassLogFile.h index fcb37ea1..5915c329 100644 --- a/code/lib/jomjol_logfile/ClassLogFile.h +++ b/code/lib/jomjol_logfile/ClassLogFile.h @@ -5,15 +5,19 @@ class ClassLogFile { private: + std::string logroot; std::string logfile; bool doLogFile; + unsigned short retentionInDays; public: - ClassLogFile(std::string _logfile); + ClassLogFile(std::string _logpath, std::string _logfile); void SwitchOnOff(bool _doLogFile); + void SetRetention(unsigned short _retentionInDays); void WriteToFile(std::string info, bool _time = true); void WriteToDedicatedFile(std::string _fn, std::string info, bool _time = true); + void RemoveOld(); }; extern ClassLogFile LogFile; \ No newline at end of file diff --git a/code/lib/jomjol_tfliteclass/CTfLiteClass._cpp_old b/code/lib/jomjol_tfliteclass/CTfLiteClass._cpp_old deleted file mode 100644 index dcd56ce0..00000000 --- a/code/lib/jomjol_tfliteclass/CTfLiteClass._cpp_old +++ /dev/null @@ -1,254 +0,0 @@ -#include "CTfLiteClass.h" - -#include "bitmap_image.hpp" - -#include - -float CTfLiteClass::GetOutputValue(int nr) -{ - TfLiteTensor* output2 = this->interpreter->output(0); - - int numeroutput = output2->dims->data[1]; - if ((nr+1) > numeroutput) - return -1000; - - return output2->data.f[nr]; -} - - -int CTfLiteClass::GetClassFromImage(std::string _fn) -{ -// printf("Before Load image %s\n", _fn.c_str()); - if (!LoadInputImage(_fn)) - return -1000; -// printf("After Load image %s\n", _fn.c_str()); - - Invoke(); - printf("After Invoke %s\n", _fn.c_str()); - - return GetOutClassification(); -// return 0; -} - -int CTfLiteClass::GetOutClassification() -{ - TfLiteTensor* output2 = interpreter->output(0); - - float zw_max = 0; - float zw; - int zw_class = -1; - - if (output2 == NULL) - return -1; - - int numeroutput = output2->dims->data[1]; - for (int i = 0; i < numeroutput; ++i) - { - zw = output2->data.f[i]; - if (zw > zw_max) - { - zw_max = zw; - zw_class = i; - } - } -// printf("Result Ziffer: %d\n", zw_class); - return zw_class; -} - -void CTfLiteClass::GetInputDimension(bool silent = false) -{ - TfLiteTensor* input2 = this->interpreter->input(0); - - int numdim = input2->dims->size; - if (!silent) printf("NumDimension: %d\n", numdim); - - int sizeofdim; - for (int j = 0; j < numdim; ++j) - { - sizeofdim = input2->dims->data[j]; - if (!silent) printf("SizeOfDimension %d: %d\n", j, sizeofdim); - if (j == 1) im_height = sizeofdim; - if (j == 2) im_width = sizeofdim; - if (j == 3) im_channel = sizeofdim; - } -} - - -void CTfLiteClass::GetOutPut() -{ - TfLiteTensor* output2 = this->interpreter->output(0); - - int numdim = output2->dims->size; - printf("NumDimension: %d\n", numdim); - - int sizeofdim; - for (int j = 0; j < numdim; ++j) - { - sizeofdim = output2->dims->data[j]; - printf("SizeOfDimension %d: %d\n", j, sizeofdim); - } - - - float fo; - - // Process the inference results. - int numeroutput = output2->dims->data[1]; - for (int i = 0; i < numeroutput; ++i) - { - fo = output2->data.f[i]; - printf("Result %d: %f\n", i, fo); - } -} - -void CTfLiteClass::Invoke() -{ - interpreter->Invoke(); -// printf("Invoke Done.\n"); -} - - -bool CTfLiteClass::LoadInputImage(std::string _fn) -{ - bitmap_image image(_fn); - unsigned int w = image.width(); - unsigned int h = image.height(); - unsigned char red, green, blue; - - input_i = 0; - float* input_data_ptr = (interpreter->input(0))->data.f; - - for (int y = 0; y < h; ++y) - for (int x = 0; x < w; ++x) - { - red = image.red_channel(x, y); - green = image.green_channel(x, y); - blue = image.blue_channel(x, y); - *(input_data_ptr) = (float) red; - input_data_ptr++; - *(input_data_ptr) = (float) green; - input_data_ptr++; - *(input_data_ptr) = (float) blue; - input_data_ptr++; - -// printf("BMP: %f %f %f\n", (float) red, (float) green, (float) blue); - - } - return true; -} - - -void CTfLiteClass::MakeAllocate() -{ -/* - this->micro_op_resolver.AddBuiltin( - tflite::BuiltinOperator_RESHAPE, - tflite::ops::micro::Register_RESHAPE()); - this->micro_op_resolver.AddBuiltin(tflite::BuiltinOperator_CONV_2D, - tflite::ops::micro::Register_CONV_2D()); - this->micro_op_resolver.AddBuiltin(tflite::BuiltinOperator_FULLY_CONNECTED, - tflite::ops::micro::Register_FULLY_CONNECTED()); - this->micro_op_resolver.AddBuiltin(tflite::BuiltinOperator_SOFTMAX, - tflite::ops::micro::Register_SOFTMAX()); - this->micro_op_resolver.AddBuiltin(tflite::BuiltinOperator_DEPTHWISE_CONV_2D, - tflite::ops::micro::Register_DEPTHWISE_CONV_2D()); - - - this->interpreter = new tflite::MicroInterpreter(this->model, this->micro_op_resolver, this->tensor_arena, this->kTensorArenaSize, this->error_reporter); -*/ - - - static tflite::ops::micro::AllOpsResolver resolver; - this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize, this->error_reporter); - - TfLiteStatus allocate_status = this->interpreter->AllocateTensors(); - if (allocate_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter, "AllocateTensors() failed"); - this->GetInputDimension(); - return; - } - - printf("Allocate Done.\n"); -} - -void CTfLiteClass::GetInputTensorSize(){ - float *zw = this->input; - int test = sizeof(zw); - printf("Input Tensor Dimension: %d\n", test); - - printf("Input Tensor Dimension: %d\n", test); -} - -long CTfLiteClass::GetFileSize(std::string filename) -{ - struct stat stat_buf; - long rc = stat(filename.c_str(), &stat_buf); - return rc == 0 ? stat_buf.st_size : -1; -} - - -unsigned char* CTfLiteClass::ReadFileToCharArray(std::string _fn) -{ - long size; - - size = this->GetFileSize(_fn); - - if (size == -1) - { - printf("\nFile existiert nicht.\n"); - return NULL; - } - - - unsigned char *result = (unsigned char*) malloc(size); - - if(result != NULL) { -// printf("\nSpeicher ist reserviert\n"); - FILE* f = fopen(_fn.c_str(), "rb"); // vorher nur "r" - fread(result, 1, size, f); - fclose(f); - }else { - printf("\nKein freier Speicher vorhanden.\n"); - } - - - return result; -} - -void CTfLiteClass::LoadModel(std::string _fn){ - - - this->error_reporter = new tflite::MicroErrorReporter; - - unsigned char *rd; - rd = this->ReadFileToCharArray(_fn.c_str()); -// printf("loadedfile: %d", (int) rd); - - this->model = tflite::GetModel(rd); - free(rd); - TFLITE_MINIMAL_CHECK(model != nullptr); - printf("tfile Loaded.\n"); - -} - - - -CTfLiteClass::CTfLiteClass() -{ -// this->accessSD = _accessSD; - this->model = nullptr; - this->interpreter = nullptr; - this->input = nullptr; - this->output = nullptr; - this->kTensorArenaSize = 600 * 1024; - this->tensor_arena = new uint8_t[kTensorArenaSize]; - -// micro_op_resolver.AddBuiltin(tflite::BuiltinOperator_CONV_2D, -// tflite::ops::micro::Register_CONV_2D()); -} - -CTfLiteClass::~CTfLiteClass() -{ - delete this->tensor_arena; -} - - diff --git a/code/lib/jomjol_tfliteclass/CTfLiteClass._h_old b/code/lib/jomjol_tfliteclass/CTfLiteClass._h_old deleted file mode 100644 index 01d972bf..00000000 --- a/code/lib/jomjol_tfliteclass/CTfLiteClass._h_old +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once - -#ifndef __CFINDTEMPLATE -#define __CFINGTEMPLATE - -#define TFLITE_MINIMAL_CHECK(x) \ - if (!(x)) { \ - fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); \ - exit(1); \ - } - -//#include "CAccessSD.h" -#include "CFindTemplate.h" - -#include "tensorflow/lite/micro/kernels/all_ops_resolver.h" -#include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/micro_interpreter.h" -#include "tensorflow/lite/schema/schema_generated.h" -#include "tensorflow/lite/version.h" -#include "tensorflow/lite/micro/kernels/micro_ops.h" -#include "esp_err.h" -#include "esp_log.h" - -//extern CAccessSDClass accessSD; - -class CTfLiteClass -{ - protected: -// CAccessSDClass *accessSD; - - tflite::ErrorReporter* error_reporter; - - const tflite::Model* model; - tflite::MicroInterpreter* interpreter; -// TfLiteTensor* input = nullptr; - TfLiteTensor* output = nullptr; - static tflite::ops::micro::AllOpsResolver *resolver; - - tflite::MicroOpResolver<5> micro_op_resolver; - - - int kTensorArenaSize; - uint8_t *tensor_arena; - - float* input; - int input_i; - - int im_height, im_width, im_channel; - - long GetFileSize(std::string filename); - unsigned char* ReadFileToCharArray(std::string _fn); - - public: -// CTfLiteClass(CAccessSDClass *_accessSD); - CTfLiteClass(); - ~CTfLiteClass(); - void LoadModel(std::string _fn); - void MakeAllocate(); - void GetInputTensorSize(); - bool LoadInputImage(std::string _fn); - void Invoke(); - void GetOutPut(); - int GetOutClassification(); - int GetClassFromImage(std::string _fn); - - float GetOutputValue(int nr); - void GetInputDimension(bool silent); - -}; - - -#endif \ No newline at end of file diff --git a/code/lib/jomjol_tfliteclass/CTfLiteClass.cpp b/code/lib/jomjol_tfliteclass/CTfLiteClass.cpp index 382f0e66..c6a2d7c5 100644 --- a/code/lib/jomjol_tfliteclass/CTfLiteClass.cpp +++ b/code/lib/jomjol_tfliteclass/CTfLiteClass.cpp @@ -152,7 +152,8 @@ bool CTfLiteClass::LoadInputImage(std::string _fn) void CTfLiteClass::MakeAllocate() { - static tflite::ops::micro::AllOpsResolver resolver; +// static tflite::ops::micro::AllOpsResolver resolver; + static tflite::AllOpsResolver resolver; this->interpreter = new tflite::MicroInterpreter(this->model, resolver, this->tensor_arena, this->kTensorArenaSize, this->error_reporter); TfLiteStatus allocate_status = this->interpreter->AllocateTensors(); @@ -243,6 +244,8 @@ CTfLiteClass::CTfLiteClass() CTfLiteClass::~CTfLiteClass() { delete this->tensor_arena; + delete this->interpreter; + delete this->error_reporter; } diff --git a/code/lib/jomjol_tfliteclass/CTfLiteClass.h b/code/lib/jomjol_tfliteclass/CTfLiteClass.h index 8beb48ba..1350a86a 100644 --- a/code/lib/jomjol_tfliteclass/CTfLiteClass.h +++ b/code/lib/jomjol_tfliteclass/CTfLiteClass.h @@ -5,7 +5,7 @@ exit(1); \ } -#include "tensorflow/lite/micro/kernels/all_ops_resolver.h" +#include "tensorflow/lite/micro/all_ops_resolver.h" #include "tensorflow/lite/micro/micro_error_reporter.h" #include "tensorflow/lite/micro/micro_interpreter.h" #include "tensorflow/lite/schema/schema_generated.h" @@ -39,7 +39,8 @@ class CTfLiteClass const tflite::Model* model; tflite::MicroInterpreter* interpreter; TfLiteTensor* output = nullptr; - static tflite::ops::micro::AllOpsResolver *resolver; +// static tflite::ops::micro::AllOpsResolver *resolver; + static tflite::AllOpsResolver resolver; int kTensorArenaSize; uint8_t *tensor_arena; diff --git a/code/lib/jomjol_time_sntp/time_sntp.cpp b/code/lib/jomjol_time_sntp/time_sntp.cpp index 92dd1876..6c8b9047 100644 --- a/code/lib/jomjol_time_sntp/time_sntp.cpp +++ b/code/lib/jomjol_time_sntp/time_sntp.cpp @@ -124,4 +124,4 @@ static void initialize_sntp(void) sntp_setservername(0, "pool.ntp.org"); sntp_set_time_sync_notification_cb(time_sync_notification_cb); sntp_init(); -} \ No newline at end of file +} diff --git a/code/lib/tfmicro/CMakeLists.txt b/code/lib/tfmicro/CMakeLists.txt index b0344ddb..f56b874c 100644 --- a/code/lib/tfmicro/CMakeLists.txt +++ b/code/lib/tfmicro/CMakeLists.txt @@ -23,8 +23,8 @@ if(NOT DEFINED ENV{IDF_PATH}) endif() idf_component_register( - SRCS tensorflow/lite/micro/simple_memory_allocator.cc tensorflow/lite/micro/micro_error_reporter.cc tensorflow/lite/micro/memory_helpers.cc tensorflow/lite/micro/test_helpers.cc tensorflow/lite/micro/micro_utils.cc tensorflow/lite/micro/micro_time.cc tensorflow/lite/micro/debug_log.cc tensorflow/lite/micro/micro_string.cc tensorflow/lite/micro/micro_optional_debug_tools.cc tensorflow/lite/micro/micro_interpreter.cc tensorflow/lite/micro/micro_allocator.cc tensorflow/lite/micro/kernels/comparisons.cc tensorflow/lite/micro/kernels/fully_connected.cc tensorflow/lite/micro/kernels/depthwise_conv.cc tensorflow/lite/micro/kernels/logistic.cc tensorflow/lite/micro/kernels/pooling.cc tensorflow/lite/micro/kernels/prelu.cc tensorflow/lite/micro/kernels/concatenation.cc tensorflow/lite/micro/kernels/dequantize.cc tensorflow/lite/micro/kernels/pad.cc tensorflow/lite/micro/kernels/l2norm.cc tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc tensorflow/lite/micro/kernels/activations.cc tensorflow/lite/micro/kernels/ceil.cc tensorflow/lite/micro/kernels/arg_min_max.cc tensorflow/lite/micro/kernels/reduce.cc tensorflow/lite/micro/kernels/unpack.cc tensorflow/lite/micro/kernels/add.cc tensorflow/lite/micro/kernels/split.cc tensorflow/lite/micro/kernels/circular_buffer.cc tensorflow/lite/micro/kernels/softmax.cc tensorflow/lite/micro/kernels/floor.cc tensorflow/lite/micro/kernels/sub.cc tensorflow/lite/micro/kernels/mul.cc tensorflow/lite/micro/kernels/conv.cc tensorflow/lite/micro/kernels/neg.cc tensorflow/lite/micro/kernels/quantize.cc tensorflow/lite/micro/kernels/elementwise.cc tensorflow/lite/micro/kernels/all_ops_resolver.cc tensorflow/lite/micro/kernels/svdf.cc tensorflow/lite/micro/kernels/maximum_minimum.cc tensorflow/lite/micro/kernels/reshape.cc tensorflow/lite/micro/kernels/strided_slice.cc tensorflow/lite/micro/kernels/round.cc tensorflow/lite/micro/kernels/pack.cc tensorflow/lite/micro/kernels/logical.cc tensorflow/lite/micro/memory_planner/linear_memory_planner.cc tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc tensorflow/lite/c/common.c tensorflow/lite/core/api/error_reporter.cc tensorflow/lite/core/api/flatbuffer_conversions.cc tensorflow/lite/core/api/op_resolver.cc tensorflow/lite/core/api/tensor_utils.cc tensorflow/lite/kernels/internal/quantization_util.cc tensorflow/lite/kernels/kernel_util.cc tensorflow/lite/micro/testing/test_utils.cc - INCLUDE_DIRS . third_party/gemmlowp third_party/flatbuffers/include third_party/ruy third_party/kissfft) + SRCS tensorflow/lite/micro/micro_error_reporter.cc tensorflow/lite/micro/simple_memory_allocator.cc tensorflow/lite/micro/memory_helpers.cc tensorflow/lite/micro/test_helpers.cc tensorflow/lite/micro/recording_micro_allocator.cc tensorflow/lite/micro/micro_time.cc tensorflow/lite/micro/recording_simple_memory_allocator.cc tensorflow/lite/micro/micro_string.cc tensorflow/lite/micro/micro_profiler.cc tensorflow/lite/micro/debug_log.cc tensorflow/lite/micro/all_ops_resolver.cc tensorflow/lite/micro/micro_utils.cc tensorflow/lite/micro/micro_interpreter.cc tensorflow/lite/micro/micro_allocator.cc tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.cc tensorflow/lite/micro/memory_planner/linear_memory_planner.cc tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc tensorflow/lite/micro/testing/test_conv_model.cc tensorflow/lite/c/common.c tensorflow/lite/core/api/error_reporter.cc tensorflow/lite/core/api/flatbuffer_conversions.cc tensorflow/lite/core/api/op_resolver.cc tensorflow/lite/core/api/tensor_utils.cc tensorflow/lite/kernels/internal/quantization_util.cc tensorflow/lite/kernels/kernel_util.cc tensorflow/lite/schema/schema_utils.cc tensorflow/lite/micro/kernels/prelu.cc tensorflow/lite/micro/kernels/dequantize.cc tensorflow/lite/micro/kernels/pad.cc tensorflow/lite/micro/kernels/shape.cc tensorflow/lite/micro/kernels/l2norm.cc tensorflow/lite/micro/kernels/tanh.cc tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc tensorflow/lite/micro/kernels/logical.cc tensorflow/lite/micro/kernels/kernel_util.cc tensorflow/lite/micro/kernels/ceil.cc tensorflow/lite/micro/kernels/arg_min_max.cc tensorflow/lite/micro/kernels/softmax.cc tensorflow/lite/micro/kernels/sub.cc tensorflow/lite/micro/kernels/add.cc tensorflow/lite/micro/kernels/floor.cc tensorflow/lite/micro/kernels/kernel_runner.cc tensorflow/lite/micro/kernels/split_v.cc tensorflow/lite/micro/kernels/hard_swish.cc tensorflow/lite/micro/kernels/pooling.cc tensorflow/lite/micro/kernels/concatenation.cc tensorflow/lite/micro/kernels/mul.cc tensorflow/lite/micro/kernels/unpack.cc tensorflow/lite/micro/kernels/round.cc tensorflow/lite/micro/kernels/quantize.cc tensorflow/lite/micro/kernels/ethosu.cc tensorflow/lite/micro/kernels/svdf.cc tensorflow/lite/micro/kernels/maximum_minimum.cc tensorflow/lite/micro/kernels/reshape.cc tensorflow/lite/micro/kernels/reduce.cc tensorflow/lite/micro/kernels/strided_slice.cc tensorflow/lite/micro/kernels/neg.cc tensorflow/lite/micro/kernels/pack.cc tensorflow/lite/micro/kernels/elementwise.cc tensorflow/lite/micro/kernels/comparisons.cc tensorflow/lite/micro/kernels/fully_connected.cc tensorflow/lite/micro/kernels/depthwise_conv.cc tensorflow/lite/micro/kernels/split.cc tensorflow/lite/micro/kernels/logistic.cc tensorflow/lite/micro/kernels/circular_buffer.cc tensorflow/lite/micro/kernels/conv.cc tensorflow/lite/micro/kernels/activations.cc + INCLUDE_DIRS . third_party/gemmlowp third_party/flatbuffers/include third_party/ruy) # Reduce the level of paranoia to be able to compile TF sources target_compile_options(${COMPONENT_LIB} PRIVATE @@ -32,6 +32,7 @@ target_compile_options(${COMPONENT_LIB} PRIVATE -Wno-missing-field-initializers -Wno-type-limits) -target_compile_options(${COMPONENT_LIB} PRIVATE -std=c11 -DTF_LITE_STATIC_MEMORY -O3 -Wno-nonnull -Wno-nonnull -Wno-nonnull -Wno-nonnull) -target_compile_options(${COMPONENT_LIB} PRIVATE $<$: -std=c++11 -DTF_LITE_STATIC_MEMORY -O3 -Wno-return-type -Wno-strict-aliasing -Wno-ignored-qualifiers -Wno-return-type -Wno-strict-aliasing -Wno-ignored-qualifiers -Wno-return-type -Wno-strict-aliasing -Wno-return-type -Wno-strict-aliasing >) +target_compile_options(${COMPONENT_LIB} PRIVATE -fno-unwind-tables -ffunction-sections -fdata-sections -fmessage-length=0 -DTF_LITE_STATIC_MEMORY -DTF_LITE_DISABLE_X86_NEON -O3 -Werror -Wsign-compare -Wdouble-promotion -Wshadow -Wunused-variable -Wmissing-field-initializers -Wunused-function -Wswitch -Wvla -Wall -Wextra -Wstrict-aliasing -Wno-unused-parameter) +target_compile_options(${COMPONENT_LIB} PRIVATE $<$: -std=c++11 -fno-rtti -fno-exceptions -fno-threadsafe-statics -fno-unwind-tables -ffunction-sections -fdata-sections -fmessage-length=0 -DTF_LITE_STATIC_MEMORY -DTF_LITE_DISABLE_X86_NEON -O3 -Werror -Wsign-compare -Wdouble-promotion -Wshadow -Wunused-variable -Wmissing-field-initializers -Wunused-function -Wswitch -Wvla -Wall -Wextra -Wstrict-aliasing -Wno-unused-parameter >) +target_compile_options(${COMPONENT_LIB} INTERFACE $<$>:-DTF_LITE_STATIC_MEMORY>) target_link_libraries(${COMPONENT_LIB} PRIVATE -lm) diff --git a/code/lib/tfmicro/third_party/gemmlowp/LICENSE b/code/lib/tfmicro/LICENSE similarity index 100% rename from code/lib/tfmicro/third_party/gemmlowp/LICENSE rename to code/lib/tfmicro/LICENSE diff --git a/code/lib/tfmicro/fixedpoint/fixedpoint_neon.h b/code/lib/tfmicro/fixedpoint/fixedpoint_neon.h new file mode 100644 index 00000000..646c5907 --- /dev/null +++ b/code/lib/tfmicro/fixedpoint/fixedpoint_neon.h @@ -0,0 +1,331 @@ +// Copyright 2015 The Gemmlowp Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// fixedpoint_neon.h: optimized NEON specializations of the templates +// in fixedpoint.h. + +#ifndef GEMMLOWP_INTERNAL_FIXEDPOINT_NEON_H_ +#define GEMMLOWP_INTERNAL_FIXEDPOINT_NEON_H_ + +#include + +namespace gemmlowp { + +template <> +struct FixedPointRawTypeTraits { + typedef std::int32_t ScalarRawType; + static constexpr int kLanes = 4; +}; + +template <> +struct FixedPointRawTypeTraits { + typedef std::int16_t ScalarRawType; + static constexpr int kLanes = 8; +}; + +template <> +inline int32x4_t BitAnd(int32x4_t a, int32x4_t b) { + return vandq_s32(a, b); +} + +template <> +inline int16x8_t BitAnd(int16x8_t a, int16x8_t b) { + return vandq_s16(a, b); +} + +template <> +inline int32x4_t BitOr(int32x4_t a, int32x4_t b) { + return vorrq_s32(a, b); +} + +template <> +inline int16x8_t BitOr(int16x8_t a, int16x8_t b) { + return vorrq_s16(a, b); +} + +template <> +inline int32x4_t BitXor(int32x4_t a, int32x4_t b) { + return veorq_s32(a, b); +} + +template <> +inline int16x8_t BitXor(int16x8_t a, int16x8_t b) { + return veorq_s16(a, b); +} + +template <> +inline int32x4_t BitNot(int32x4_t a) { + return veorq_s32(a, vdupq_n_s32(-1)); +} + +template <> +inline int16x8_t BitNot(int16x8_t a) { + return veorq_s16(a, vdupq_n_s16(-1)); +} + +template <> +inline int32x4_t Add(int32x4_t a, int32x4_t b) { + return vaddq_s32(a, b); +} + +template <> +inline int16x8_t Add(int16x8_t a, int16x8_t b) { + return vaddq_s16(a, b); +} + +template <> +inline int32x4_t Sub(int32x4_t a, int32x4_t b) { + return vsubq_s32(a, b); +} + +template <> +inline int16x8_t Sub(int16x8_t a, int16x8_t b) { + return vsubq_s16(a, b); +} + +template <> +inline int32x4_t Neg(int32x4_t a) { + return vnegq_s32(a); +} + +template <> +inline int16x8_t Neg(int16x8_t a) { + return vnegq_s16(a); +} + +template <> +inline int32x4_t ShiftLeft(int32x4_t a, int offset) { + return vshlq_s32(a, vdupq_n_s32(offset)); +} + +template <> +inline int16x8_t ShiftLeft(int16x8_t a, int offset) { + return vshlq_s16(a, vdupq_n_s16(offset)); +} + +template <> +inline int32x4_t ShiftRight(int32x4_t a, int offset) { + return vshlq_s32(a, vdupq_n_s32(-offset)); +} + +template <> +inline int16x8_t ShiftRight(int16x8_t a, int offset) { + return vshlq_s16(a, vdupq_n_s16(-offset)); +} + +template <> +inline int32x4_t SelectUsingMask(int32x4_t if_mask, int32x4_t then_val, + int32x4_t else_val) { + return vbslq_s32(vreinterpretq_u32_s32(if_mask), then_val, else_val); +} + +template <> +inline int16x8_t SelectUsingMask(int16x8_t if_mask, int16x8_t then_val, + int16x8_t else_val) { + return vbslq_s16(vreinterpretq_u16_s16(if_mask), then_val, else_val); +} + +template <> +inline int32x4_t MaskIfEqual(int32x4_t a, int32x4_t b) { + return vreinterpretq_s32_u32(vceqq_s32(a, b)); +} + +template <> +inline int16x8_t MaskIfEqual(int16x8_t a, int16x8_t b) { + return vreinterpretq_s16_u16(vceqq_s16(a, b)); +} + +template <> +inline int32x4_t MaskIfNotEqual(int32x4_t a, int32x4_t b) { + return BitNot(MaskIfEqual(a, b)); +} + +template <> +inline int16x8_t MaskIfNotEqual(int16x8_t a, int16x8_t b) { + return BitNot(MaskIfEqual(a, b)); +} + +template <> +inline int32x4_t MaskIfZero(int32x4_t a) { + return MaskIfEqual(a, vdupq_n_s32(0)); +} + +template <> +inline int16x8_t MaskIfZero(int16x8_t a) { + return MaskIfEqual(a, vdupq_n_s16(0)); +} + +template <> +inline int32x4_t MaskIfNonZero(int32x4_t a) { + return vreinterpretq_s32_u32(vtstq_s32(a, a)); +} + +template <> +inline int16x8_t MaskIfNonZero(int16x8_t a) { + return vreinterpretq_s16_u16(vtstq_s16(a, a)); +} + +template <> +inline int32x4_t MaskIfGreaterThan(int32x4_t a, int32x4_t b) { + return vreinterpretq_s32_u32(vcgtq_s32(a, b)); +} + +template <> +inline int16x8_t MaskIfGreaterThan(int16x8_t a, int16x8_t b) { + return vreinterpretq_s16_u16(vcgtq_s16(a, b)); +} + +template <> +inline int32x4_t MaskIfGreaterThanOrEqual(int32x4_t a, int32x4_t b) { + return vreinterpretq_s32_u32(vcgeq_s32(a, b)); +} + +template <> +inline int16x8_t MaskIfGreaterThanOrEqual(int16x8_t a, int16x8_t b) { + return vreinterpretq_s16_u16(vcgeq_s16(a, b)); +} + +template <> +inline int32x4_t MaskIfLessThan(int32x4_t a, int32x4_t b) { + return vreinterpretq_s32_u32(vcltq_s32(a, b)); +} + +template <> +inline int16x8_t MaskIfLessThan(int16x8_t a, int16x8_t b) { + return vreinterpretq_s16_u16(vcltq_s16(a, b)); +} + +template <> +inline int32x4_t MaskIfLessThanOrEqual(int32x4_t a, int32x4_t b) { + return vreinterpretq_s32_u32(vcleq_s32(a, b)); +} + +template <> +inline int16x8_t MaskIfLessThanOrEqual(int16x8_t a, int16x8_t b) { + return vreinterpretq_s16_u16(vcleq_s16(a, b)); +} + +template <> +inline bool All(int32x4_t a) { + a = vandq_s32(a, vextq_s32(a, a, 1)); + a = vandq_s32(a, vextq_s32(a, a, 2)); + return vgetq_lane_s32(a, 0); +} + +template <> +inline bool All(int16x8_t a) { + a = vandq_s16(a, vextq_s16(a, a, 1)); + a = vandq_s16(a, vextq_s16(a, a, 2)); + a = vandq_s16(a, vextq_s16(a, a, 4)); + return vgetq_lane_s16(a, 0); +} + +template <> +inline bool Any(int32x4_t a) { + a = vorrq_s32(a, vextq_s32(a, a, 1)); + a = vorrq_s32(a, vextq_s32(a, a, 2)); + return vgetq_lane_s32(a, 0); +} + +template <> +inline bool Any(int16x8_t a) { + a = vorrq_s16(a, vextq_s16(a, a, 1)); + a = vorrq_s16(a, vextq_s16(a, a, 2)); + a = vorrq_s16(a, vextq_s16(a, a, 4)); + return vgetq_lane_s16(a, 0); +} + +template <> +inline int32x4_t RoundingHalfSum(int32x4_t a, int32x4_t b) { + return vrhaddq_s32(a, b); +} + +template <> +inline int16x8_t RoundingHalfSum(int16x8_t a, int16x8_t b) { + return vrhaddq_s16(a, b); +} + +template <> +inline int32x4_t SaturatingRoundingDoublingHighMul(int32x4_t a, int32x4_t b) { + return vqrdmulhq_s32(a, b); +} + +template <> +inline int16x8_t SaturatingRoundingDoublingHighMul(int16x8_t a, int16x8_t b) { + return vqrdmulhq_s16(a, b); +} + +template <> +inline int32x4_t RoundingDivideByPOT(int32x4_t x, int exponent) { + const int32x4_t shift_vec = vdupq_n_s32(-exponent); + const int32x4_t fixup = vshrq_n_s32(vandq_s32(x, shift_vec), 31); + const int32x4_t fixed_up_x = vqaddq_s32(x, fixup); + return vrshlq_s32(fixed_up_x, shift_vec); +} + +template <> +inline int16x8_t RoundingDivideByPOT(int16x8_t x, int exponent) { + const int16x8_t shift_vec = vdupq_n_s16(-exponent); + const int16x8_t fixup = vshrq_n_s16(vandq_s16(x, shift_vec), 15); + const int16x8_t fixed_up_x = vqaddq_s16(x, fixup); + return vrshlq_s16(fixed_up_x, shift_vec); +} + +template +struct ImplSaturatingRoundingMultiplyByPOT { + static int32x4_t eval(int32x4_t x) { return vqshlq_n_s32(x, Exponent); } +}; + +template +struct ImplSaturatingRoundingMultiplyByPOT { + static int32x4_t eval(int32x4_t x) { + const int32x4_t fixup = vshrq_n_s32(x, 31); + const int32x4_t fixed_up_x = vqaddq_s32(x, fixup); + return vrshrq_n_s32(fixed_up_x, -Exponent); + } +}; + +template +struct ImplSaturatingRoundingMultiplyByPOT { + static int16x8_t eval(int16x8_t x) { return vqshlq_n_s16(x, Exponent); } +}; + +template +struct ImplSaturatingRoundingMultiplyByPOT { + static int16x8_t eval(int16x8_t x) { + const int16x8_t fixup = vshrq_n_s16(x, 15); + const int16x8_t fixed_up_x = vqaddq_s16(x, fixup); + return vrshrq_n_s16(fixed_up_x, -Exponent); + } +}; + +template <> +inline int32x4_t Dup(std::int32_t x) { + return vdupq_n_s32(x); +} + +template <> +inline int16x8_t Dup(std::int16_t x) { + return vdupq_n_s16(x); +} + +// So far this is only needed for int16. +template <> +inline int16x8_t SaturatingAdd(int16x8_t a, int16x8_t b) { + return vqaddq_s16(a, b); +} + +} // end namespace gemmlowp + +#endif // GEMMLOWP_INTERNAL_FIXEDPOINT_NEON_H_ diff --git a/code/lib/tfmicro/third_party/flatbuffers/LICENSE.txt b/code/lib/tfmicro/flatbuffers/LICENSE.txt similarity index 100% rename from code/lib/tfmicro/third_party/flatbuffers/LICENSE.txt rename to code/lib/tfmicro/flatbuffers/LICENSE.txt diff --git a/code/lib/tfmicro/flatbuffers/base.h b/code/lib/tfmicro/flatbuffers/base.h index 95573806..8e97c084 100644 --- a/code/lib/tfmicro/flatbuffers/base.h +++ b/code/lib/tfmicro/flatbuffers/base.h @@ -46,14 +46,17 @@ #include #include +#if defined(__unix__) && !defined(FLATBUFFERS_LOCALE_INDEPENDENT) + #include +#endif + #ifdef _STLPORT_VERSION #define FLATBUFFERS_CPP98_STL #endif -#ifndef FLATBUFFERS_CPP98_STL - #include -#endif -#include "flatbuffers/stl_emulation.h" +#ifdef __ANDROID__ + #include +#endif #if defined(__ICCARM__) #include @@ -154,10 +157,12 @@ namespace flatbuffers { defined(__clang__) #define FLATBUFFERS_FINAL_CLASS final #define FLATBUFFERS_OVERRIDE override + #define FLATBUFFERS_EXPLICIT_CPP11 explicit #define FLATBUFFERS_VTABLE_UNDERLYING_TYPE : flatbuffers::voffset_t #else #define FLATBUFFERS_FINAL_CLASS #define FLATBUFFERS_OVERRIDE + #define FLATBUFFERS_EXPLICIT_CPP11 #define FLATBUFFERS_VTABLE_UNDERLYING_TYPE #endif @@ -165,10 +170,14 @@ namespace flatbuffers { (!defined(__GNUC__) || (__GNUC__ * 100 + __GNUC_MINOR__ >= 406)) || \ (defined(__cpp_constexpr) && __cpp_constexpr >= 200704) #define FLATBUFFERS_CONSTEXPR constexpr + #define FLATBUFFERS_CONSTEXPR_CPP11 constexpr + #define FLATBUFFERS_CONSTEXPR_DEFINED #else #define FLATBUFFERS_CONSTEXPR const + #define FLATBUFFERS_CONSTEXPR_CPP11 #endif +// This macro is never used in code! #if (defined(__cplusplus) && __cplusplus >= 201402L) || \ (defined(__cpp_constexpr) && __cpp_constexpr >= 201304) #define FLATBUFFERS_CONSTEXPR_CPP14 FLATBUFFERS_CONSTEXPR @@ -194,6 +203,16 @@ namespace flatbuffers { #define FLATBUFFERS_DELETE_FUNC(func) private: func; #endif +// Check if we can use template aliases +// Not possible if Microsoft Compiler before 2012 +// Possible is the language feature __cpp_alias_templates is defined well +// Or possible if the C++ std is C+11 or newer +#if (defined(_MSC_VER) && _MSC_VER > 1700 /* MSVC2012 */) \ + || (defined(__cpp_alias_templates) && __cpp_alias_templates >= 200704) \ + || (defined(__cplusplus) && __cplusplus >= 201103L) + #define FLATBUFFERS_TEMPLATES_ALIASES +#endif + #ifndef FLATBUFFERS_HAS_STRING_VIEW // Only provide flatbuffers::string_view if __has_include can be used // to detect a header that provides an implementation @@ -236,10 +255,8 @@ namespace flatbuffers { #ifndef FLATBUFFERS_LOCALE_INDEPENDENT // Enable locale independent functions {strtof_l, strtod_l,strtoll_l, strtoull_l}. - // They are part of the POSIX-2008 but not part of the C/C++ standard. - // GCC/Clang have definition (_XOPEN_SOURCE>=700) if POSIX-2008. #if ((defined(_MSC_VER) && _MSC_VER >= 1800) || \ - (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE>=700))) + (defined(_XOPEN_VERSION) && (_XOPEN_VERSION>=700)) && (!defined(__ANDROID_API__) || (defined(__ANDROID_API__) && (__ANDROID_API__>=21)))) #define FLATBUFFERS_LOCALE_INDEPENDENT 1 #else #define FLATBUFFERS_LOCALE_INDEPENDENT 0 @@ -309,6 +326,7 @@ typedef uintmax_t largest_scalar_t; #define FLATBUFFERS_MAX_ALIGNMENT 16 #if defined(_MSC_VER) + #pragma warning(disable: 4351) // C4351: new behavior: elements of array ... will be default initialized #pragma warning(push) #pragma warning(disable: 4127) // C4127: conditional expression is constant #endif @@ -374,6 +392,13 @@ T ReadScalar(const void *p) { return EndianScalar(*reinterpret_cast(p)); } +// See https://github.com/google/flatbuffers/issues/5950 + +#if (FLATBUFFERS_GCC >= 100000) && (FLATBUFFERS_GCC < 110000) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstringop-overflow" +#endif + template // UBSAN: C++ aliasing type rules, see std::bit_cast<> for details. __supress_ubsan__("alignment") @@ -386,6 +411,10 @@ template __supress_ubsan__("alignment") void WriteScalar(void *p, Of *reinterpret_cast(p) = EndianScalar(t.o); } +#if (FLATBUFFERS_GCC >= 100000) && (FLATBUFFERS_GCC < 110000) + #pragma GCC diagnostic pop +#endif + // Computes how many bytes you'd have to pad to be able to write an // "scalar_size" scalar if the buffer had grown to "buf_size" (downwards in // memory). diff --git a/code/lib/tfmicro/flatbuffers/flatbuffers.h b/code/lib/tfmicro/flatbuffers/flatbuffers.h index c4dc5bcd..799f6478 100644 --- a/code/lib/tfmicro/flatbuffers/flatbuffers.h +++ b/code/lib/tfmicro/flatbuffers/flatbuffers.h @@ -18,6 +18,11 @@ #define FLATBUFFERS_H_ #include "flatbuffers/base.h" +#include "flatbuffers/stl_emulation.h" + +#ifndef FLATBUFFERS_CPP98_STL + #include +#endif #if defined(FLATBUFFERS_NAN_DEFAULTS) # include @@ -581,6 +586,14 @@ static inline const char *GetCstring(const String *str) { return str ? str->c_str() : ""; } +#ifdef FLATBUFFERS_HAS_STRING_VIEW +// Convenience function to get string_view from a String returning an empty +// string_view on null pointer. +static inline flatbuffers::string_view GetStringView(const String *str) { + return str ? str->string_view() : flatbuffers::string_view(); +} +#endif // FLATBUFFERS_HAS_STRING_VIEW + // Allocator interface. This is flatbuffers-specific and meant only for // `vector_downward` usage. class Allocator { @@ -1211,7 +1224,7 @@ class FlatBufferBuilder { /// you call Finish()). You can use this information if you need to embed /// a FlatBuffer in some other buffer, such that you can later read it /// without first having to copy it into its own buffer. - size_t GetBufferMinAlignment() { + size_t GetBufferMinAlignment() const { Finished(); return minalign_; } @@ -1295,6 +1308,11 @@ class FlatBufferBuilder { TrackField(field, off); } + template void AddElement(voffset_t field, T e) { + auto off = PushElement(e); + TrackField(field, off); + } + template void AddOffset(voffset_t field, Offset off) { if (off.IsNull()) return; // Don't store. AddElement(field, ReferTo(off.o), static_cast(0)); @@ -1599,6 +1617,9 @@ class FlatBufferBuilder { // causing the wrong overload to be selected, remove it. AssertScalarT(); StartVector(len, sizeof(T)); + if (len == 0) { + return Offset>(EndVector(len)); + } // clang-format off #if FLATBUFFERS_LITTLEENDIAN PushBytes(reinterpret_cast(v), len * sizeof(T)); @@ -1795,8 +1816,8 @@ class FlatBufferBuilder { return a.KeyCompareLessThan(&b); } - private: - StructKeyComparator &operator=(const StructKeyComparator &); + FLATBUFFERS_DELETE_FUNC( + StructKeyComparator &operator=(const StructKeyComparator &)) }; /// @endcond @@ -1871,10 +1892,7 @@ class FlatBufferBuilder { vector_downward &buf_; private: - TableKeyComparator &operator=(const TableKeyComparator &other) { - buf_ = other.buf_; - return *this; - } + FLATBUFFERS_DELETE_FUNC(TableKeyComparator &operator=(const TableKeyComparator &other)) }; /// @endcond @@ -2269,8 +2287,8 @@ class Verifier FLATBUFFERS_FINAL_CLASS { template bool VerifyBufferFromStart(const char *identifier, size_t start) { - if (identifier && (size_ < 2 * sizeof(flatbuffers::uoffset_t) || - !BufferHasIdentifier(buf_ + start, identifier))) { + if (identifier && !Check((size_ >= 2 * sizeof(flatbuffers::uoffset_t) && + BufferHasIdentifier(buf_ + start, identifier)))) { return false; } @@ -2452,12 +2470,26 @@ class Table { return field_offset ? reinterpret_cast

(p) : nullptr; } + template + flatbuffers::Optional GetOptional(voffset_t field) const { + auto field_offset = GetOptionalFieldOffset(field); + auto p = data_ + field_offset; + return field_offset ? Optional(static_cast(ReadScalar(p))) + : Optional(); + } + template bool SetField(voffset_t field, T val, T def) { auto field_offset = GetOptionalFieldOffset(field); if (!field_offset) return IsTheSameAs(val, def); WriteScalar(data_ + field_offset, val); return true; } + template bool SetField(voffset_t field, T val) { + auto field_offset = GetOptionalFieldOffset(field); + if (!field_offset) return false; + WriteScalar(data_ + field_offset, val); + return true; + } bool SetPointer(voffset_t field, const uint8_t *val) { auto field_offset = GetOptionalFieldOffset(field); @@ -2525,6 +2557,17 @@ class Table { uint8_t data_[1]; }; +// This specialization allows avoiding warnings like: +// MSVC C4800: type: forcing value to bool 'true' or 'false'. +template<> +inline flatbuffers::Optional Table::GetOptional( + voffset_t field) const { + auto field_offset = GetOptionalFieldOffset(field); + auto p = data_ + field_offset; + return field_offset ? Optional(ReadScalar(p) != 0) + : Optional(); +} + template void FlatBufferBuilder::Required(Offset table, voffset_t field) { auto table_ptr = reinterpret_cast(buf_.data_at(table.o)); @@ -2704,7 +2747,7 @@ inline const char * const *ElementaryTypeNames() { // Basic type info cost just 16bits per field! struct TypeCode { uint16_t base_type : 4; // ElementaryType - uint16_t is_vector : 1; + uint16_t is_repeating : 1; // Either vector (in table) or array (in struct) int16_t sequence_ref : 11; // Index into type_refs below, or -1 for none. }; @@ -2720,6 +2763,7 @@ struct TypeTable { size_t num_elems; // of type_codes, values, names (but not type_refs). const TypeCode *type_codes; // num_elems count const TypeFunction *type_refs; // less than num_elems entries (see TypeCode). + const int16_t *array_sizes; // less than num_elems entries (see TypeCode). const int64_t *values; // Only set for non-consecutive enum/union or structs. const char *const *names; // Only set if compiled with --reflect-names. }; diff --git a/code/lib/tfmicro/flatbuffers/stl_emulation.h b/code/lib/tfmicro/flatbuffers/stl_emulation.h index 8bae61bf..c9a1a8bf 100644 --- a/code/lib/tfmicro/flatbuffers/stl_emulation.h +++ b/code/lib/tfmicro/flatbuffers/stl_emulation.h @@ -18,6 +18,7 @@ #define FLATBUFFERS_STL_EMULATION_H_ // clang-format off +#include "flatbuffers/base.h" #include #include @@ -25,6 +26,17 @@ #include #include +// Detect C++17 compatible compiler. +// __cplusplus >= 201703L - a compiler has support of 'static inline' variables. +#if defined(FLATBUFFERS_USE_STD_OPTIONAL) \ + || (defined(__cplusplus) && __cplusplus >= 201703L) \ + || (defined(_MSVC_LANG) && (_MSVC_LANG >= 201703L)) + #include + #ifndef FLATBUFFERS_USE_STD_OPTIONAL + #define FLATBUFFERS_USE_STD_OPTIONAL + #endif +#endif + #if defined(_STLPORT_VERSION) && !defined(FLATBUFFERS_CPP98_STL) #define FLATBUFFERS_CPP98_STL #endif // defined(_STLPORT_VERSION) && !defined(FLATBUFFERS_CPP98_STL) @@ -33,16 +45,6 @@ #include #endif // defined(FLATBUFFERS_CPP98_STL) -// Check if we can use template aliases -// Not possible if Microsoft Compiler before 2012 -// Possible is the language feature __cpp_alias_templates is defined well -// Or possible if the C++ std is C+11 or newer -#if (defined(_MSC_VER) && _MSC_VER > 1700 /* MSVC2012 */) \ - || (defined(__cpp_alias_templates) && __cpp_alias_templates >= 200704) \ - || (defined(__cplusplus) && __cplusplus >= 201103L) - #define FLATBUFFERS_TEMPLATES_ALIASES -#endif - // This header provides backwards compatibility for C++98 STLs like stlport. namespace flatbuffers { @@ -190,7 +192,7 @@ inline void vector_emplace_back(std::vector *vector, V &&data) { // MSVC 2010 doesn't support C++11 aliases. // We're manually "aliasing" the class here as we want to bring unique_ptr // into the flatbuffers namespace. We have unique_ptr in the flatbuffers - // namespace we have a completely independent implemenation (see below) + // namespace we have a completely independent implementation (see below) // for C++98 STL implementations. template class unique_ptr : public std::unique_ptr { public: @@ -302,6 +304,146 @@ inline void vector_emplace_back(std::vector *vector, V &&data) { #endif // !FLATBUFFERS_CPP98_STL +#ifdef FLATBUFFERS_USE_STD_OPTIONAL +template +using Optional = std::optional; +using nullopt_t = std::nullopt_t; +inline constexpr nullopt_t nullopt = std::nullopt; + +#else +// Limited implementation of Optional type for a scalar T. +// This implementation limited by trivial types compatible with +// std::is_arithmetic or std::is_enum type traits. + +// A tag to indicate an empty flatbuffers::optional. +struct nullopt_t { + explicit FLATBUFFERS_CONSTEXPR_CPP11 nullopt_t(int) {} +}; + +#if defined(FLATBUFFERS_CONSTEXPR_DEFINED) + namespace internal { + template struct nullopt_holder { + static constexpr nullopt_t instance_ = nullopt_t(0); + }; + template + constexpr nullopt_t nullopt_holder::instance_; + } + static constexpr const nullopt_t &nullopt = internal::nullopt_holder::instance_; + +#else + namespace internal { + template struct nullopt_holder { + static const nullopt_t instance_; + }; + template + const nullopt_t nullopt_holder::instance_ = nullopt_t(0); + } + static const nullopt_t &nullopt = internal::nullopt_holder::instance_; + +#endif + +template +class Optional FLATBUFFERS_FINAL_CLASS { + // Non-scalar 'T' would extremely complicated Optional. + // Use is_scalar checking because flatbuffers flatbuffers::is_arithmetic + // isn't implemented. + static_assert(flatbuffers::is_scalar::value, "unexpected type T"); + + public: + ~Optional() {} + + FLATBUFFERS_CONSTEXPR_CPP11 Optional() FLATBUFFERS_NOEXCEPT + : value_(), has_value_(false) {} + + FLATBUFFERS_CONSTEXPR_CPP11 Optional(nullopt_t) FLATBUFFERS_NOEXCEPT + : value_(), has_value_(false) {} + + FLATBUFFERS_CONSTEXPR_CPP11 Optional(T val) FLATBUFFERS_NOEXCEPT + : value_(val), has_value_(true) {} + + FLATBUFFERS_CONSTEXPR_CPP11 Optional(const Optional &other) FLATBUFFERS_NOEXCEPT + : value_(other.value_), has_value_(other.has_value_) {} + + FLATBUFFERS_CONSTEXPR_CPP14 Optional &operator=(const Optional &other) FLATBUFFERS_NOEXCEPT { + value_ = other.value_; + has_value_ = other.has_value_; + return *this; + } + + FLATBUFFERS_CONSTEXPR_CPP14 Optional &operator=(nullopt_t) FLATBUFFERS_NOEXCEPT { + value_ = T(); + has_value_ = false; + return *this; + } + + FLATBUFFERS_CONSTEXPR_CPP14 Optional &operator=(T val) FLATBUFFERS_NOEXCEPT { + value_ = val; + has_value_ = true; + return *this; + } + + void reset() FLATBUFFERS_NOEXCEPT { + *this = nullopt; + } + + void swap(Optional &other) FLATBUFFERS_NOEXCEPT { + std::swap(value_, other.value_); + std::swap(has_value_, other.has_value_); + } + + FLATBUFFERS_CONSTEXPR_CPP11 FLATBUFFERS_EXPLICIT_CPP11 operator bool() const FLATBUFFERS_NOEXCEPT { + return has_value_; + } + + FLATBUFFERS_CONSTEXPR_CPP11 bool has_value() const FLATBUFFERS_NOEXCEPT { + return has_value_; + } + + FLATBUFFERS_CONSTEXPR_CPP11 const T& operator*() const FLATBUFFERS_NOEXCEPT { + return value_; + } + + const T& value() const { + FLATBUFFERS_ASSERT(has_value()); + return value_; + } + + T value_or(T default_value) const FLATBUFFERS_NOEXCEPT { + return has_value() ? value_ : default_value; + } + + private: + T value_; + bool has_value_; +}; + +template +FLATBUFFERS_CONSTEXPR_CPP11 bool operator==(const Optional& opt, nullopt_t) FLATBUFFERS_NOEXCEPT { + return !opt; +} +template +FLATBUFFERS_CONSTEXPR_CPP11 bool operator==(nullopt_t, const Optional& opt) FLATBUFFERS_NOEXCEPT { + return !opt; +} + +template +FLATBUFFERS_CONSTEXPR_CPP11 bool operator==(const Optional& lhs, const U& rhs) FLATBUFFERS_NOEXCEPT { + return static_cast(lhs) && (*lhs == rhs); +} + +template +FLATBUFFERS_CONSTEXPR_CPP11 bool operator==(const T& lhs, const Optional& rhs) FLATBUFFERS_NOEXCEPT { + return static_cast(rhs) && (lhs == *rhs); +} + +template +FLATBUFFERS_CONSTEXPR_CPP11 bool operator==(const Optional& lhs, const Optional& rhs) FLATBUFFERS_NOEXCEPT { + return static_cast(lhs) != static_cast(rhs) + ? false + : !static_cast(lhs) ? false : (*lhs == *rhs); +} +#endif // FLATBUFFERS_USE_STD_OPTIONAL + } // namespace flatbuffers #endif // FLATBUFFERS_STL_EMULATION_H_ diff --git a/code/lib/tfmicro/kissfft/COPYING b/code/lib/tfmicro/kissfft/COPYING deleted file mode 100644 index 2fc6685a..00000000 --- a/code/lib/tfmicro/kissfft/COPYING +++ /dev/null @@ -1,11 +0,0 @@ -Copyright (c) 2003-2010 Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/code/lib/tfmicro/kissfft/_kiss_fft_guts.h b/code/lib/tfmicro/kissfft/_kiss_fft_guts.h deleted file mode 100644 index ba661444..00000000 --- a/code/lib/tfmicro/kissfft/_kiss_fft_guts.h +++ /dev/null @@ -1,164 +0,0 @@ -/* -Copyright (c) 2003-2010, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/* kiss_fft.h - defines kiss_fft_scalar as either short or a float type - and defines - typedef struct { kiss_fft_scalar r; kiss_fft_scalar i; }kiss_fft_cpx; */ -#include "kiss_fft.h" -#include - -#define MAXFACTORS 32 -/* e.g. an fft of length 128 has 4 factors - as far as kissfft is concerned - 4*4*4*2 - */ - -struct kiss_fft_state{ - int nfft; - int inverse; - int factors[2*MAXFACTORS]; - kiss_fft_cpx twiddles[1]; -}; - -/* - Explanation of macros dealing with complex math: - - C_MUL(m,a,b) : m = a*b - C_FIXDIV( c , div ) : if a fixed point impl., c /= div. noop otherwise - C_SUB( res, a,b) : res = a - b - C_SUBFROM( res , a) : res -= a - C_ADDTO( res , a) : res += a - * */ -#ifdef FIXED_POINT -#if (FIXED_POINT==32) -# define FRACBITS 31 -# define SAMPPROD int64_t -#define SAMP_MAX 2147483647 -#else -# define FRACBITS 15 -# define SAMPPROD int32_t -#define SAMP_MAX 32767 -#endif - -#define SAMP_MIN -SAMP_MAX - -#if defined(CHECK_OVERFLOW) -# define CHECK_OVERFLOW_OP(a,op,b) \ - if ( (SAMPPROD)(a) op (SAMPPROD)(b) > SAMP_MAX || (SAMPPROD)(a) op (SAMPPROD)(b) < SAMP_MIN ) { \ - fprintf(stderr,"WARNING:overflow @ " __FILE__ "(%d): (%d " #op" %d) = %ld\n",__LINE__,(a),(b),(SAMPPROD)(a) op (SAMPPROD)(b) ); } -#endif - - -# define smul(a,b) ( (SAMPPROD)(a)*(b) ) -# define sround( x ) (kiss_fft_scalar)( ( (x) + (1<<(FRACBITS-1)) ) >> FRACBITS ) - -# define S_MUL(a,b) sround( smul(a,b) ) - -# define C_MUL(m,a,b) \ - do{ (m).r = sround( smul((a).r,(b).r) - smul((a).i,(b).i) ); \ - (m).i = sround( smul((a).r,(b).i) + smul((a).i,(b).r) ); }while(0) - -# define DIVSCALAR(x,k) \ - (x) = sround( smul( x, SAMP_MAX/k ) ) - -# define C_FIXDIV(c,div) \ - do { DIVSCALAR( (c).r , div); \ - DIVSCALAR( (c).i , div); }while (0) - -# define C_MULBYSCALAR( c, s ) \ - do{ (c).r = sround( smul( (c).r , s ) ) ;\ - (c).i = sround( smul( (c).i , s ) ) ; }while(0) - -#else /* not FIXED_POINT*/ - -# define S_MUL(a,b) ( (a)*(b) ) -#define C_MUL(m,a,b) \ - do{ (m).r = (a).r*(b).r - (a).i*(b).i;\ - (m).i = (a).r*(b).i + (a).i*(b).r; }while(0) -# define C_FIXDIV(c,div) /* NOOP */ -# define C_MULBYSCALAR( c, s ) \ - do{ (c).r *= (s);\ - (c).i *= (s); }while(0) -#endif - -#ifndef CHECK_OVERFLOW_OP -# define CHECK_OVERFLOW_OP(a,op,b) /* noop */ -#endif - -#define C_ADD( res, a,b)\ - do { \ - CHECK_OVERFLOW_OP((a).r,+,(b).r)\ - CHECK_OVERFLOW_OP((a).i,+,(b).i)\ - (res).r=(a).r+(b).r; (res).i=(a).i+(b).i; \ - }while(0) -#define C_SUB( res, a,b)\ - do { \ - CHECK_OVERFLOW_OP((a).r,-,(b).r)\ - CHECK_OVERFLOW_OP((a).i,-,(b).i)\ - (res).r=(a).r-(b).r; (res).i=(a).i-(b).i; \ - }while(0) -#define C_ADDTO( res , a)\ - do { \ - CHECK_OVERFLOW_OP((res).r,+,(a).r)\ - CHECK_OVERFLOW_OP((res).i,+,(a).i)\ - (res).r += (a).r; (res).i += (a).i;\ - }while(0) - -#define C_SUBFROM( res , a)\ - do {\ - CHECK_OVERFLOW_OP((res).r,-,(a).r)\ - CHECK_OVERFLOW_OP((res).i,-,(a).i)\ - (res).r -= (a).r; (res).i -= (a).i; \ - }while(0) - - -#ifdef FIXED_POINT -# define KISS_FFT_COS(phase) floor(.5+SAMP_MAX * cos (phase)) -# define KISS_FFT_SIN(phase) floor(.5+SAMP_MAX * sin (phase)) -# define HALF_OF(x) ((x)>>1) -#elif defined(USE_SIMD) -# define KISS_FFT_COS(phase) _mm_set1_ps( cos(phase) ) -# define KISS_FFT_SIN(phase) _mm_set1_ps( sin(phase) ) -# define HALF_OF(x) ((x)*_mm_set1_ps(.5)) -#else -# define KISS_FFT_COS(phase) (kiss_fft_scalar) cos(phase) -# define KISS_FFT_SIN(phase) (kiss_fft_scalar) sin(phase) -# define HALF_OF(x) ((x)*.5) -#endif - -#define kf_cexp(x,phase) \ - do{ \ - (x)->r = KISS_FFT_COS(phase);\ - (x)->i = KISS_FFT_SIN(phase);\ - }while(0) - - -/* a debugging function */ -#define pcpx(c)\ - fprintf(stderr,"%g + %gi\n",(double)((c)->r),(double)((c)->i) ) - - -#ifdef KISS_FFT_USE_ALLOCA -// define this to allow use of alloca instead of malloc for temporary buffers -// Temporary buffers are used in two case: -// 1. FFT sizes that have "bad" factors. i.e. not 2,3 and 5 -// 2. "in-place" FFTs. Notice the quotes, since kissfft does not really do an in-place transform. -#include -#define KISS_FFT_TMP_ALLOC(nbytes) alloca(nbytes) -#define KISS_FFT_TMP_FREE(ptr) -#else -#define KISS_FFT_TMP_ALLOC(nbytes) KISS_FFT_MALLOC(nbytes) -#define KISS_FFT_TMP_FREE(ptr) KISS_FFT_FREE(ptr) -#endif diff --git a/code/lib/tfmicro/kissfft/kiss_fft.h b/code/lib/tfmicro/kissfft/kiss_fft.h deleted file mode 100644 index c34ea5ee..00000000 --- a/code/lib/tfmicro/kissfft/kiss_fft.h +++ /dev/null @@ -1,131 +0,0 @@ -#ifndef KISS_FFT_H -#define KISS_FFT_H - -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* - ATTENTION! - If you would like a : - -- a utility that will handle the caching of fft objects - -- real-only (no imaginary time component ) FFT - -- a multi-dimensional FFT - -- a command-line utility to perform ffts - -- a command-line utility to perform fast-convolution filtering - - Then see kfc.h kiss_fftr.h kiss_fftnd.h fftutil.c kiss_fastfir.c - in the tools/ directory. -*/ - -#ifdef USE_SIMD -# include -# define kiss_fft_scalar __m128 -#define KISS_FFT_MALLOC(nbytes) _mm_malloc(nbytes,16) -#define KISS_FFT_FREE _mm_free -#else -#define KISS_FFT_MALLOC(X) (void*)(0) /* Patched. */ -#define KISS_FFT_FREE(X) /* Patched. */ -#endif - - -// Patched automatically by download_dependencies.sh so default is 16 bit. -#ifndef FIXED_POINT -#define FIXED_POINT (16) -#endif -// End patch. - -#ifdef FIXED_POINT -#include /* Patched. */ -#include -# if (FIXED_POINT == 32) -# define kiss_fft_scalar int32_t -# else -# define kiss_fft_scalar int16_t -# endif -#else -# ifndef kiss_fft_scalar -/* default is float */ -# define kiss_fft_scalar float -# endif -#endif - -typedef struct { - kiss_fft_scalar r; - kiss_fft_scalar i; -}kiss_fft_cpx; - -typedef struct kiss_fft_state* kiss_fft_cfg; - -/* - * kiss_fft_alloc - * - * Initialize a FFT (or IFFT) algorithm's cfg/state buffer. - * - * typical usage: kiss_fft_cfg mycfg=kiss_fft_alloc(1024,0,NULL,NULL); - * - * The return value from fft_alloc is a cfg buffer used internally - * by the fft routine or NULL. - * - * If lenmem is NULL, then kiss_fft_alloc will allocate a cfg buffer using malloc. - * The returned value should be free()d when done to avoid memory leaks. - * - * The state can be placed in a user supplied buffer 'mem': - * If lenmem is not NULL and mem is not NULL and *lenmem is large enough, - * then the function places the cfg in mem and the size used in *lenmem - * and returns mem. - * - * If lenmem is not NULL and ( mem is NULL or *lenmem is not large enough), - * then the function returns NULL and places the minimum cfg - * buffer size in *lenmem. - * */ - -kiss_fft_cfg kiss_fft_alloc(int nfft,int inverse_fft,void * mem,size_t * lenmem); - -/* - * kiss_fft(cfg,in_out_buf) - * - * Perform an FFT on a complex input buffer. - * for a forward FFT, - * fin should be f[0] , f[1] , ... ,f[nfft-1] - * fout will be F[0] , F[1] , ... ,F[nfft-1] - * Note that each element is complex and can be accessed like - f[k].r and f[k].i - * */ -void kiss_fft(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout); - -/* - A more generic version of the above function. It reads its input from every Nth sample. - * */ -void kiss_fft_stride(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout,int fin_stride); - -/* If kiss_fft_alloc allocated a buffer, it is one contiguous - buffer and can be simply free()d when no longer needed*/ -#define kiss_fft_free free - -/* - Cleans up some memory that gets managed internally. Not necessary to call, but it might clean up - your compiler output to call this before you exit. -*/ -void kiss_fft_cleanup(void); - - -/* - * Returns the smallest integer k, such that k>=n and k has only "fast" factors (2,3,5) - */ -int kiss_fft_next_fast_size(int n); - -/* for real ffts, we need an even size */ -#define kiss_fftr_next_fast_size_real(n) \ - (kiss_fft_next_fast_size( ((n)+1)>>1)<<1) - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/code/lib/tfmicro/kissfft/tools/kiss_fftr.h b/code/lib/tfmicro/kissfft/tools/kiss_fftr.h deleted file mode 100644 index 72e5a577..00000000 --- a/code/lib/tfmicro/kissfft/tools/kiss_fftr.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef KISS_FTR_H -#define KISS_FTR_H - -#include "kiss_fft.h" -#ifdef __cplusplus -extern "C" { -#endif - - -/* - - Real optimized version can save about 45% cpu time vs. complex fft of a real seq. - - - - */ - -typedef struct kiss_fftr_state *kiss_fftr_cfg; - - -kiss_fftr_cfg kiss_fftr_alloc(int nfft,int inverse_fft,void * mem, size_t * lenmem); -/* - nfft must be even - - If you don't care to allocate space, use mem = lenmem = NULL -*/ - - -void kiss_fftr(kiss_fftr_cfg cfg,const kiss_fft_scalar *timedata,kiss_fft_cpx *freqdata); -/* - input timedata has nfft scalar points - output freqdata has nfft/2+1 complex points -*/ - -void kiss_fftri(kiss_fftr_cfg cfg,const kiss_fft_cpx *freqdata,kiss_fft_scalar *timedata); -/* - input freqdata has nfft/2+1 complex points - output timedata has nfft scalar points -*/ - -#define kiss_fftr_free free - -#ifdef __cplusplus -} -#endif -#endif diff --git a/code/lib/tfmicro/ruy/ruy/profiler/instrumentation.h b/code/lib/tfmicro/ruy/ruy/profiler/instrumentation.h deleted file mode 100644 index c4df1e68..00000000 --- a/code/lib/tfmicro/ruy/ruy/profiler/instrumentation.h +++ /dev/null @@ -1,203 +0,0 @@ -/* Copyright 2020 Google LLC. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef RUY_RUY_PROFILER_INSTRUMENTATION_H_ -#define RUY_RUY_PROFILER_INSTRUMENTATION_H_ - -#ifdef RUY_PROFILER -#include -#include -#include -#endif - -namespace ruy { -namespace profiler { - -#ifdef RUY_PROFILER - -// A label is how a code scope is annotated to appear in profiles. -// The stacks that are sampled by the profiler are stacks of such labels. -// A label consists of a literal string, plus optional integer arguments. -class Label { - public: - Label() {} - template - explicit Label(Args... args) { - Set(args...); - } - void Set(const char* format) { - format_ = format; - args_count_ = 0; - } - template - void Set(const char* format, Args... args) { - format_ = format; - args_count_ = sizeof...(args); - SetArgs(0, args...); - } - - void operator=(const Label& other); - - bool operator==(const Label& other) const; - - std::string Formatted() const; - const char* format() const { return format_; } - - private: - void SetArgs(int position, int arg0) { args_[position] = arg0; } - - template - void SetArgs(int position, int arg0, Args... args) { - SetArgs(position, arg0); - SetArgs(position + 1, args...); - } - - static constexpr int kMaxArgs = 4; - const char* format_ = nullptr; - int args_count_ = 0; - int args_[kMaxArgs]; -}; - -namespace detail { - -// Forward-declaration, see class ThreadStack below. -class ThreadStack; - -bool& GlobalIsProfilerRunning(); - -// Returns the global vector of pointers to all stacks, there being one stack -// per thread executing instrumented code. -std::vector* GlobalAllThreadStacks(); - -// Returns the mutex to be locked around any access to GlobalAllThreadStacks(). -std::mutex* GlobalsMutex(); - -// Returns the thread-local stack, specific to the current thread. -ThreadStack* ThreadLocalThreadStack(); - -// This 'stack' is what may be more appropriately called a 'pseudostack': -// It contains Label entries that are 'manually' entered by instrumentation -// code. It's unrelated to real call stacks. -struct Stack { - std::uint32_t id = 0; - static constexpr int kMaxSize = 64; - int size = 0; - Label labels[kMaxSize]; -}; - -// Returns the buffer byte size required by CopyToSample. -int GetBufferSize(const Stack& stack); - -// Copies this Stack into a byte buffer, called a 'sample'. -void CopyToBuffer(const Stack& stack, char* dst); - -// Populates this Stack from an existing sample buffer, typically -// produced by CopyToSample. -void ReadFromBuffer(const char* src, Stack* stack); - -// ThreadStack is meant to be used as a thread-local singleton, assigning to -// each thread a Stack object holding its pseudo-stack of profile labels, -// plus a mutex allowing to synchronize accesses to this pseudo-stack between -// this thread and a possible profiler thread sampling it. -class ThreadStack { - public: - ThreadStack(); - ~ThreadStack(); - - const Stack& stack() const { return stack_; } - - // Returns the mutex to lock around any access to this stack. Each stack is - // accessed by potentially two threads: the thread that it belongs to - // (which calls Push and Pop) and the profiler thread during profiling - // (which calls CopyToSample). - std::mutex& Mutex() const { return mutex_; } - - // Pushes a new label on the top of this Stack. - template - void Push(Args... args) { - // This mutex locking is needed to guard against race conditions as both - // the current thread and the profiler thread may be concurrently accessing - // this stack. In addition to that, this mutex locking also serves the other - // purpose of acting as a barrier (of compiler code reordering, of runtime - // CPU instruction reordering, and of memory access reordering), which - // gives a measure of correctness to this profiler. The downside is some - // latency. As this lock will be uncontended most of the times, the cost - // should be roughly that of an sequentially-consistent atomic access, - // comparable to an access to the level of CPU data cache that is shared - // among all cores, typically 60 cycles on current ARM CPUs, plus side - // effects from barrier instructions. - std::lock_guard lock(mutex_); - // Avoid overrunning the stack, even in 'release' builds. This profiling - // instrumentation code should not ship in release builds anyway, the - // overhead of this check is negligible, and overrunning a stack array would - // be bad. - if (stack_.size >= Stack::kMaxSize) { - abort(); - } - stack_.labels[stack_.size++].Set(args...); - } - - // Pops the top-most label from this Stack. - void Pop() { - // See the comment in Push about this lock. While it would be tempting to - // try to remove this lock and just atomically decrement size_ with a - // store-release, that would not necessarily be a substitute for all of the - // purposes that this lock serves, or if it was done carefully to serve all - // of the same purposes, then that wouldn't be faster than this (mostly - // uncontended) lock. - std::lock_guard lock(mutex_); - stack_.size--; - } - - private: - mutable std::mutex mutex_; - Stack stack_; -}; - -} // namespace detail - -// RAII user-facing way to construct Labels associated with their life scope -// and get them pushed to / popped from the current thread stack. -class ScopeLabel { - public: - template - ScopeLabel(Args... args) : thread_stack_(detail::ThreadLocalThreadStack()) { - thread_stack_->Push(args...); - } - - ~ScopeLabel() { thread_stack_->Pop(); } - - private: - detail::ThreadStack* thread_stack_; -}; - -#else // no RUY_PROFILER - -class ScopeLabel { - public: - template - explicit ScopeLabel(Args...) {} - - // This destructor is needed to consistently silence clang's -Wunused-variable - // which seems to trigger semi-randomly. - ~ScopeLabel() {} -}; - -#endif - -} // namespace profiler -} // namespace ruy - -#endif // RUY_RUY_PROFILER_INSTRUMENTATION_H_ diff --git a/code/lib/tfmicro/tensorflow/core/public/version.h b/code/lib/tfmicro/tensorflow/core/public/version.h index 8f983022..08318293 100644 --- a/code/lib/tfmicro/tensorflow/core/public/version.h +++ b/code/lib/tfmicro/tensorflow/core/public/version.h @@ -21,7 +21,7 @@ limitations under the License. // Also update tensorflow/tensorflow.bzl and // tensorflow/tools/pip_package/setup.py #define TF_MAJOR_VERSION 2 -#define TF_MINOR_VERSION 1 +#define TF_MINOR_VERSION 5 #define TF_PATCH_VERSION 0 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 389 // Updated: 2020/5/2 +#define TF_GRAPH_DEF_VERSION 578 // Updated: 2020/11/7 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // diff --git a/code/lib/tfmicro/tensorflow/lite/c/builtin_op_data.h b/code/lib/tfmicro/tensorflow/lite/c/builtin_op_data.h index 2fe6c053..5452ef63 100644 --- a/code/lib/tfmicro/tensorflow/lite/c/builtin_op_data.h +++ b/code/lib/tfmicro/tensorflow/lite/c/builtin_op_data.h @@ -67,8 +67,9 @@ typedef struct { typedef enum { kTfLiteActNone = 0, kTfLiteActRelu, - kTfLiteActRelu1, // min(max(-1, x), 1) - kTfLiteActRelu6, // min(max(0, x), 6) + kTfLiteActReluN1To1, // min(max(-1, x), 1) + kTfLiteActRelu1 = kTfLiteActReluN1To1, // kTfLiteActRelu1 will be deprecated. + kTfLiteActRelu6, // min(max(0, x), 6) kTfLiteActTanh, kTfLiteActSignBit, kTfLiteActSigmoid, @@ -198,6 +199,8 @@ typedef struct { typedef struct { TfLiteFusedActivation activation; + // Parameter added for the version 4. + bool pot_scale_int16; } TfLiteAddParams; typedef struct { @@ -219,6 +222,8 @@ typedef struct { typedef struct { TfLiteFusedActivation activation; + // Parameter added for the version 5. + bool pot_scale_int16; } TfLiteSubParams; typedef struct { @@ -297,6 +302,7 @@ typedef struct { typedef struct { bool align_corners; + bool half_pixel_centers; } TfLiteResizeNearestNeighborParams; typedef struct { @@ -459,6 +465,15 @@ typedef struct { int body_subgraph_index; } TfLiteWhileParams; +typedef struct { + bool exclusive; + bool reverse; +} TfLiteCumsumParams; + +typedef struct { + int init_subgraph_index; +} TfLiteCallOnceParams; + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/code/lib/tfmicro/tensorflow/lite/c/common.c b/code/lib/tfmicro/tensorflow/lite/c/common.c index f70a6000..0264f420 100644 --- a/code/lib/tfmicro/tensorflow/lite/c/common.c +++ b/code/lib/tfmicro/tensorflow/lite/c/common.c @@ -79,7 +79,8 @@ TfLiteFloatArray* TfLiteFloatArrayCreate(int size) { void TfLiteFloatArrayFree(TfLiteFloatArray* a) { free(a); } void TfLiteTensorDataFree(TfLiteTensor* t) { - if (t->allocation_type == kTfLiteDynamic) { + if (t->allocation_type == kTfLiteDynamic || + t->allocation_type == kTfLitePersistentRo) { free(t->data.raw); } t->data.raw = NULL; @@ -172,7 +173,8 @@ void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims, } void TfLiteTensorRealloc(size_t num_bytes, TfLiteTensor* tensor) { - if (tensor->allocation_type != kTfLiteDynamic) { + if (tensor->allocation_type != kTfLiteDynamic && + tensor->allocation_type != kTfLitePersistentRo) { return; } // TODO(b/145340303): Tensor data should be aligned. @@ -205,6 +207,8 @@ const char* TfLiteTypeGetName(TfLiteType type) { return "BOOL"; case kTfLiteComplex64: return "COMPLEX64"; + case kTfLiteComplex128: + return "COMPLEX128"; case kTfLiteString: return "STRING"; case kTfLiteFloat16: diff --git a/code/lib/tfmicro/tensorflow/lite/c/common.h b/code/lib/tfmicro/tensorflow/lite/c/common.h index 12ddf994..e04e1a12 100644 --- a/code/lib/tfmicro/tensorflow/lite/c/common.h +++ b/code/lib/tfmicro/tensorflow/lite/c/common.h @@ -29,6 +29,9 @@ limitations under the License. // TfLiteDelegate - allows delegation of nodes to alternative backends. // // Some abstractions in this file are created and managed by Interpreter. +// +// NOTE: The order of values in these structs are "semi-ABI stable". New values +// should be added only to the end of structs and never reordered. #ifndef TENSORFLOW_LITE_C_COMMON_H_ #define TENSORFLOW_LITE_C_COMMON_H_ @@ -43,8 +46,18 @@ extern "C" { typedef enum TfLiteStatus { kTfLiteOk = 0, + + // Generally referring to an error in the runtime (i.e. interpreter) kTfLiteError = 1, - kTfLiteDelegateError = 2 + + // Generally referring to an error from a TfLiteDelegate itself. + kTfLiteDelegateError = 2, + + // Generally referring to an error in applying a delegate due to + // incompatibility between runtime and delegate, e.g., this error is returned + // when trying to apply a TfLite delegate onto a model graph that's already + // immutable. + kTfLiteApplicationError = 3 } TfLiteStatus; // The list of external context types known to TF Lite. This list exists solely @@ -55,7 +68,7 @@ typedef enum TfLiteExternalContextType { kTfLiteEigenContext = 0, // include eigen_support.h to use. kTfLiteGemmLowpContext = 1, // include gemm_support.h to use. kTfLiteEdgeTpuContext = 2, // Placeholder for Edge TPU support. - kTfLiteCpuBackendContext = 3, // include cpu_backend_support.h to use. + kTfLiteCpuBackendContext = 3, // include cpu_backend_context.h to use. kTfLiteMaxExternalContexts = 4 } TfLiteExternalContextType; @@ -83,8 +96,9 @@ typedef struct TfLiteIntArray { int size; // gcc 6.1+ have a bug where flexible members aren't properly handled // https://github.com/google/re2/commit/b94b7cd42e9f02673cd748c1ac1d16db4052514c -#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 6 && \ - __GNUC_MINOR__ >= 1 +#if (!defined(__clang__) && defined(__GNUC__) && __GNUC__ == 6 && \ + __GNUC_MINOR__ >= 1) || \ + defined(HEXAGON) || (__clang_major__ == 7 && __clang_minor__ == 1) int data[0]; #else int data[]; @@ -122,6 +136,7 @@ typedef struct TfLiteFloatArray { int size; // gcc 6.1+ have a bug where flexible members aren't properly handled // https://github.com/google/re2/commit/b94b7cd42e9f02673cd748c1ac1d16db4052514c +// This also applies to the toolchain used for Qualcomm Hexagon DSPs. #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 6 && \ __GNUC_MINOR__ >= 1 float data[0]; @@ -200,6 +215,7 @@ void TfLiteFloatArrayFree(TfLiteFloatArray* a); // the current function, while also reporting the location of the error. // `a` and `b` may be evaluated more than once, so no side effects or // extremely expensive computations should be done. +// NOTE: Use TF_LITE_ENSURE_TYPES_EQ if comparing TfLiteTypes. #define TF_LITE_ENSURE_EQ(context, a, b) \ do { \ if ((a) != (b)) { \ @@ -219,6 +235,17 @@ void TfLiteFloatArrayFree(TfLiteFloatArray* a); } \ } while (0) +#define TF_LITE_ENSURE_NEAR(context, a, b, epsilon) \ + do { \ + auto delta = ((a) > (b)) ? ((a) - (b)) : ((b) - (a)); \ + if (delta > epsilon) { \ + TF_LITE_KERNEL_LOG((context), "%s:%d %s not near %s (%f != %f)", \ + __FILE__, __LINE__, #a, #b, static_cast(a), \ + static_cast(b)); \ + return kTfLiteError; \ + } \ + } while (0) + #define TF_LITE_ENSURE_OK(context, status) \ do { \ const TfLiteStatus s = (status); \ @@ -227,11 +254,32 @@ void TfLiteFloatArrayFree(TfLiteFloatArray* a); } \ } while (0) +// Define TFL_CAPI_EXPORT macro to export a function properly with a shared +// library. +#ifdef SWIG +#define TFL_CAPI_EXPORT +#else +#if defined(_WIN32) +#ifdef TFL_COMPILE_LIBRARY +#define TFL_CAPI_EXPORT __declspec(dllexport) +#else +#define TFL_CAPI_EXPORT __declspec(dllimport) +#endif // TFL_COMPILE_LIBRARY +#else +#define TFL_CAPI_EXPORT __attribute__((visibility("default"))) +#endif // _WIN32 +#endif // SWIG + // Single-precision complex data type compatible with the C99 definition. typedef struct TfLiteComplex64 { float re, im; // real and imaginary parts, respectively. } TfLiteComplex64; +// Double-precision complex data type compatible with the C99 definition. +typedef struct TfLiteComplex128 { + double re, im; // real and imaginary parts, respectively. +} TfLiteComplex128; + // Half precision data type compatible with the C99 definition. typedef struct TfLiteFloat16 { uint16_t data; @@ -251,6 +299,7 @@ typedef enum { kTfLiteInt8 = 9, kTfLiteFloat16 = 10, kTfLiteFloat64 = 11, + kTfLiteComplex128 = 12, } TfLiteType; // Return the name of a given type, for error reporting purposes. @@ -307,26 +356,39 @@ typedef union TfLitePtrUnion { int64_t* i64; float* f; TfLiteFloat16* f16; + double* f64; char* raw; const char* raw_const; uint8_t* uint8; bool* b; int16_t* i16; TfLiteComplex64* c64; + TfLiteComplex128* c128; int8_t* int8; /* Only use this member. */ void* data; } TfLitePtrUnion; -// Memory allocation strategies. kTfLiteMmapRo is for read-only memory-mapped -// data (or data externally allocated). kTfLiteArenaRw is arena allocated -// data. kTfLiteDynamic is for tensors that are allocated during evaluation. +// Memory allocation strategies. +// * kTfLiteMmapRo: Read-only memory-mapped data, or data externally allocated. +// * kTfLiteArenaRw: Arena allocated with no guarantees about persistence, +// and available during eval. +// * kTfLiteArenaRwPersistent: Arena allocated but persistent across eval, and +// only available during eval. +// * kTfLiteDynamic: Allocated during eval, or for string tensors. +// * kTfLitePersistentRo: Allocated and populated during prepare. This is +// useful for tensors that can be computed during prepare and treated +// as constant inputs for downstream ops (also in prepare). +// * kTfLiteCustom: Custom memory allocation provided by the user. See +// TfLiteCustomAllocation below. typedef enum TfLiteAllocationType { kTfLiteMemNone = 0, kTfLiteMmapRo, kTfLiteArenaRw, kTfLiteArenaRwPersistent, kTfLiteDynamic, + kTfLitePersistentRo, + kTfLiteCustom, } TfLiteAllocationType; // The delegates should use zero or positive integers to represent handles. @@ -359,8 +421,18 @@ typedef struct TfLiteSparsity { int dim_metadata_size; } TfLiteSparsity; -// An tensor in the interpreter system which is a wrapper around a buffer of +// Defines a custom memory allocation not owned by the runtime. +// `data` should be aligned to kDefaultTensorAlignment defined in +// lite/util.h. (Currently 64 bytes) +// NOTE: See Interpreter.SetCustomAllocationForTensor for details on usage. +typedef struct TfLiteCustomAllocation { + void* data; + size_t bytes; +} TfLiteCustomAllocation; + +// A tensor in the interpreter system which is a wrapper around a buffer of // data including a dimensionality (or NULL if not currently defined). +#ifndef TF_LITE_STATIC_MEMORY typedef struct TfLiteTensor { // The data type specification for data stored in `data`. This affects // what member of `data` union should be used. @@ -426,31 +498,6 @@ typedef struct TfLiteTensor { const TfLiteIntArray* dims_signature; } TfLiteTensor; -#ifndef TF_LITE_STATIC_MEMORY -// Free data memory of tensor `t`. -void TfLiteTensorDataFree(TfLiteTensor* t); - -// Free quantization data. -void TfLiteQuantizationFree(TfLiteQuantization* quantization); - -// Free sparsity parameters. -void TfLiteSparsityFree(TfLiteSparsity* sparsity); - -// Free memory of tensor `t`. -void TfLiteTensorFree(TfLiteTensor* t); - -// Set all of a tensor's fields (and free any previously allocated data). -void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims, - TfLiteQuantizationParams quantization, char* buffer, - size_t size, TfLiteAllocationType allocation_type, - const void* allocation, bool is_variable, - TfLiteTensor* tensor); - -// Resize the allocated data of a (dynamic) tensor. Tensors with allocation -// types other than kTfLiteDynamic will be ignored. -void TfLiteTensorRealloc(size_t num_bytes, TfLiteTensor* tensor); -#endif // TF_LITE_STATIC_MEMORY - // A structure representing an instance of a node. // This structure only exhibits the inputs, outputs and user defined data, not // other features like the type. @@ -487,6 +534,130 @@ typedef struct TfLiteNode { // WARNING: This is an experimental interface that is subject to change. struct TfLiteDelegate* delegate; } TfLiteNode; +#else // defined(TF_LITE_STATIC_MEMORY)? +// NOTE: This flag is opt-in only at compile time. +// +// Specific reduced TfLiteTensor struct for TF Micro runtime. This struct +// contains only the minimum fields required to initialize and prepare a micro +// inference graph. The fields in this struct have been ordered from +// largest-to-smallest for optimal struct sizeof. +// +// This struct does not use: +// - allocation +// - buffer_handle +// - data_is_stale +// - delegate +// - dims_signature +// - name +// - sparsity +typedef struct TfLiteTensor { + // TODO(b/155784997): Consider consolidating these quantization fields: + // Quantization information. Replaces params field above. + TfLiteQuantization quantization; + + // Quantization information. + TfLiteQuantizationParams params; + + // A union of data pointers. The appropriate type should be used for a typed + // tensor based on `type`. + TfLitePtrUnion data; + + // A pointer to a structure representing the dimensionality interpretation + // that the buffer should have. NOTE: the product of elements of `dims` + // and the element datatype size should be equal to `bytes` below. + TfLiteIntArray* dims; + + // The number of bytes required to store the data of this Tensor. I.e. + // (bytes of each element) * dims[0] * ... * dims[n-1]. For example, if + // type is kTfLiteFloat32 and dims = {3, 2} then + // bytes = sizeof(float) * 3 * 2 = 4 * 3 * 2 = 24. + size_t bytes; + + // The data type specification for data stored in `data`. This affects + // what member of `data` union should be used. + TfLiteType type; + + // How memory is mapped + // kTfLiteMmapRo: Memory mapped read only. + // i.e. weights + // kTfLiteArenaRw: Arena allocated read write memory + // (i.e. temporaries, outputs). + TfLiteAllocationType allocation_type; + + // True if the tensor is a variable. + bool is_variable; +} TfLiteTensor; + +// Specific reduced TfLiteNode struct for TF Micro runtime. This struct contains +// only the minimum fields required to represent a node. +// +// This struct does not use: +// - delegate +// - intermediates +// - temporaries +typedef struct TfLiteNode { + // Inputs to this node expressed as indices into the simulator's tensors. + TfLiteIntArray* inputs; + + // Outputs to this node expressed as indices into the simulator's tensors. + TfLiteIntArray* outputs; + + // Opaque data provided by the node implementer through `Registration.init`. + void* user_data; + + // Opaque data provided to the node if the node is a builtin. This is usually + // a structure defined in builtin_op_data.h + void* builtin_data; + + // Custom initial data. This is the opaque data provided in the flatbuffer. + // WARNING: This is an experimental interface that is subject to change. + const void* custom_initial_data; + int custom_initial_data_size; +} TfLiteNode; +#endif // TF_LITE_STATIC_MEMORY + +// Light-weight tensor struct for TF Micro runtime. Provides the minimal amount +// of information required for a kernel to run during TfLiteRegistration::Eval. +// TODO(b/160955687): Move this field into TF_LITE_STATIC_MEMORY when TFLM +// builds with this flag by default internally. +typedef struct TfLiteEvalTensor { + // A union of data pointers. The appropriate type should be used for a typed + // tensor based on `type`. + TfLitePtrUnion data; + + // A pointer to a structure representing the dimensionality interpretation + // that the buffer should have. + TfLiteIntArray* dims; + + // The data type specification for data stored in `data`. This affects + // what member of `data` union should be used. + TfLiteType type; +} TfLiteEvalTensor; + +#ifndef TF_LITE_STATIC_MEMORY +// Free data memory of tensor `t`. +void TfLiteTensorDataFree(TfLiteTensor* t); + +// Free quantization data. +void TfLiteQuantizationFree(TfLiteQuantization* quantization); + +// Free sparsity parameters. +void TfLiteSparsityFree(TfLiteSparsity* sparsity); + +// Free memory of tensor `t`. +void TfLiteTensorFree(TfLiteTensor* t); + +// Set all of a tensor's fields (and free any previously allocated data). +void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims, + TfLiteQuantizationParams quantization, char* buffer, + size_t size, TfLiteAllocationType allocation_type, + const void* allocation, bool is_variable, + TfLiteTensor* tensor); + +// Resize the allocated data of a (dynamic) tensor. Tensors with allocation +// types other than kTfLiteDynamic will be ignored. +void TfLiteTensorRealloc(size_t num_bytes, TfLiteTensor* tensor); +#endif // TF_LITE_STATIC_MEMORY // WARNING: This is an experimental interface that is subject to change. // @@ -578,12 +749,11 @@ typedef struct TfLiteContext { void* profiler; // Allocate persistent buffer which has the same life time as the interpreter. + // Returns nullptr on failure. // The memory is allocated from heap for TFL, and from tail in TFLM. - // If *ptr is not nullptr, the pointer will be reallocated. - // This method is only available in Prepare stage. + // This method is only available in Init or Prepare stage. // WARNING: This is an experimental interface that is subject to change. - TfLiteStatus (*AllocatePersistentBuffer)(struct TfLiteContext* ctx, - size_t bytes, void** ptr); + void* (*AllocatePersistentBuffer)(struct TfLiteContext* ctx, size_t bytes); // Allocate a buffer which will be deallocated right after invoke phase. // The memory is allocated from heap in TFL, and from volatile arena in TFLM. @@ -638,6 +808,18 @@ typedef struct TfLiteContext { TfLiteStatus (*PreviewDelegatePartitioning)( struct TfLiteContext* context, const TfLiteIntArray* nodes_to_replace, TfLiteDelegateParams** partition_params_array, int* num_partitions); + + // Returns a TfLiteTensor struct for a given index. + // WARNING: This is an experimental interface that is subject to change. + // WARNING: This method may not be available on all platforms. + TfLiteTensor* (*GetTensor)(const struct TfLiteContext* context, + int tensor_idx); + + // Returns a TfLiteEvalTensor struct for a given index. + // WARNING: This is an experimental interface that is subject to change. + // WARNING: This method may not be available on all platforms. + TfLiteEvalTensor* (*GetEvalTensor)(const struct TfLiteContext* context, + int tensor_idx); } TfLiteContext; typedef struct TfLiteRegistration { @@ -712,7 +894,26 @@ typedef enum TfLiteDelegateFlags { // // If the delegate isn't capable to handle dynamic tensors, this flag need // to be set to false. - kTfLiteDelegateFlagsAllowDynamicTensors = 1 + kTfLiteDelegateFlagsAllowDynamicTensors = 1, + + // This flag can be used by delegates (that allow dynamic tensors) to ensure + // applicable tensor shapes are automatically propagated in the case of tensor + // resizing. + // This means that non-dynamic (allocation_type != kTfLiteDynamic) I/O tensors + // of a delegate kernel will have correct shapes before its Prepare() method + // is called. The runtime leverages TFLite builtin ops in the original + // execution plan to propagate shapes. + // + // A few points to note: + // 1. This requires kTfLiteDelegateFlagsAllowDynamicTensors. If that flag is + // false, this one is redundant since the delegate kernels are re-initialized + // every time tensors are resized. + // 2. Enabling this flag adds some overhead to AllocateTensors(), since extra + // work is required to prepare the original execution plan. + // 3. This flag requires that the original execution plan only have ops with + // valid registrations (and not 'dummy' custom ops like with Flex). + // WARNING: This feature is experimental and subject to change. + kTfLiteDelegateFlagsRequirePropagatedShapes = 2 } TfLiteDelegateFlags; // WARNING: This is an experimental interface that is subject to change. @@ -731,8 +932,9 @@ typedef struct TfLiteDelegate { struct TfLiteDelegate* delegate); // Copy the data from delegate buffer handle into raw memory of the given - // 'tensor'. This cannot be null. The delegate is allowed to allocate the raw - // bytes as long as it follows the rules for kTfLiteDynamic tensors. + // 'tensor'. Note that the delegate is allowed to allocate the raw bytes as + // long as it follows the rules for kTfLiteDynamic tensors, in which case this + // cannot be null. TfLiteStatus (*CopyFromBufferHandle)(TfLiteContext* context, struct TfLiteDelegate* delegate, TfLiteBufferHandle buffer_handle, diff --git a/code/lib/tfmicro/tensorflow/lite/core/api/flatbuffer_conversions.cc b/code/lib/tfmicro/tensorflow/lite/core/api/flatbuffer_conversions.cc index 2325513d..16118d41 100644 --- a/code/lib/tfmicro/tensorflow/lite/core/api/flatbuffer_conversions.cc +++ b/code/lib/tfmicro/tensorflow/lite/core/api/flatbuffer_conversions.cc @@ -15,10 +15,15 @@ limitations under the License. #include "tensorflow/lite/core/api/flatbuffer_conversions.h" -#include +#include +#include +#include +#include "flatbuffers/flatbuffers.h" // from @flatbuffers #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/schema/schema_generated.h" namespace tflite { @@ -57,6 +62,17 @@ class SafeBuiltinDataAllocator { BuiltinDataAllocator* allocator_; }; +// All the Parse functions take some pointers as params and this function has +// the common DCHECKs to catch if any of those are nullptr. +void CheckParsePointerParams(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + TFLITE_DCHECK(op != nullptr); + TFLITE_DCHECK(error_reporter != nullptr); + TFLITE_DCHECK(allocator != nullptr); + TFLITE_DCHECK(builtin_data != nullptr); +} + // Copies the contents from the flatbuffer int vector `flatbuffer` into the // int array `buffer`. `flat_vector` and `buffer` represent the same // configuration operation for a given operation. @@ -85,87 +101,41 @@ TfLiteStatus FlatBufferIntVectorToArray( return kTfLiteOk; } -} // namespace - -TfLiteStatus ConvertTensorType(TensorType tensor_type, TfLiteType* type, - ErrorReporter* error_reporter) { - switch (tensor_type) { - case TensorType_FLOAT16: - *type = kTfLiteFloat16; - return kTfLiteOk; - case TensorType_FLOAT32: - *type = kTfLiteFloat32; - return kTfLiteOk; - case TensorType_FLOAT64: - *type = kTfLiteFloat64; - return kTfLiteOk; - case TensorType_INT16: - *type = kTfLiteInt16; - return kTfLiteOk; - case TensorType_INT32: - *type = kTfLiteInt32; - return kTfLiteOk; - case TensorType_UINT8: - *type = kTfLiteUInt8; - return kTfLiteOk; - case TensorType_INT8: - *type = kTfLiteInt8; - return kTfLiteOk; - case TensorType_INT64: - *type = kTfLiteInt64; - return kTfLiteOk; - case TensorType_STRING: - *type = kTfLiteString; - return kTfLiteOk; - case TensorType_BOOL: - *type = kTfLiteBool; - return kTfLiteOk; - case TensorType_COMPLEX64: - *type = kTfLiteComplex64; - return kTfLiteOk; - default: - *type = kTfLiteNoType; - TF_LITE_REPORT_ERROR(error_reporter, - "Unsupported data type %d in tensor\n", tensor_type); - return kTfLiteError; +// Converts the flatbuffer activation to what is used at runtime. +TfLiteFusedActivation ConvertActivation(ActivationFunctionType activation) { + switch (activation) { + case ActivationFunctionType_NONE: + return kTfLiteActNone; + case ActivationFunctionType_RELU: + return kTfLiteActRelu; + case ActivationFunctionType_RELU_N1_TO_1: + return kTfLiteActReluN1To1; + case ActivationFunctionType_RELU6: + return kTfLiteActRelu6; + case ActivationFunctionType_TANH: + return kTfLiteActTanh; + case ActivationFunctionType_SIGN_BIT: + return kTfLiteActSignBit; } + return kTfLiteActNone; } -// Parse the appropriate data out of the op. -// -// This handles builtin data explicitly as there are flatbuffer schemas. -// If it returns kTfLiteOk, it passes the data out with `builtin_data`, which -// need to be released by calling `free`.` -// If it returns kTfLiteError, `builtin_data` will be `nullptr`. -TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, - ErrorReporter* error_reporter, - BuiltinDataAllocator* allocator, void** builtin_data) { - auto parse_padding = [](Padding padding) { - switch (padding) { - case Padding_SAME: - return kTfLitePaddingSame; - case Padding_VALID: - return kTfLitePaddingValid; - } - return kTfLitePaddingUnknown; - }; - auto parse_activation = [](ActivationFunctionType activation) { - switch (activation) { - case ActivationFunctionType_NONE: - return kTfLiteActNone; - case ActivationFunctionType_RELU: - return kTfLiteActRelu; - case ActivationFunctionType_RELU_N1_TO_1: - return kTfLiteActRelu1; - case ActivationFunctionType_RELU6: - return kTfLiteActRelu6; - case ActivationFunctionType_TANH: - return kTfLiteActTanh; - case ActivationFunctionType_SIGN_BIT: - return kTfLiteActSignBit; - } - return kTfLiteActNone; - }; +// Converts the flatbuffer padding enum to what is used at runtime. +TfLitePadding ConvertPadding(Padding padding) { + switch (padding) { + case Padding_SAME: + return kTfLitePaddingSame; + case Padding_VALID: + return kTfLitePaddingValid; + } + return kTfLitePaddingUnknown; +} + +#ifndef TF_LITE_STATIC_MEMORY +TfLiteStatus ParseOpDataTfLite(const Operator* op, BuiltinOperator op_type, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { auto parseLSHProjectionType = [](LSHProjectionType type) { switch (type) { case LSHProjectionType_SPARSE: @@ -191,22 +161,247 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, SafeBuiltinDataAllocator safe_allocator(allocator); *builtin_data = nullptr; switch (op_type) { - case BuiltinOperator_CONV_2D: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (auto* conv_params = op->builtin_options_as_Conv2DOptions()) { - params->padding = parse_padding(conv_params->padding()); - params->stride_width = conv_params->stride_w(); - params->stride_height = conv_params->stride_h(); - params->activation = - parse_activation(conv_params->fused_activation_function()); - - params->dilation_width_factor = conv_params->dilation_w_factor(); - params->dilation_height_factor = conv_params->dilation_h_factor(); - } - *builtin_data = params.release(); - return kTfLiteOk; + case BuiltinOperator_ABS: { + return ParseAbs(op, error_reporter, allocator, builtin_data); } + + case BuiltinOperator_ADD: { + return ParseAdd(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_ARG_MAX: { + return ParseArgMax(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_ARG_MIN: { + return ParseArgMin(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_AVERAGE_POOL_2D: { + return ParsePool(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_CEIL: { + return ParseCeil(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_CONCATENATION: { + return ParseConcatenation(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_CONV_2D: { + return ParseConv2D(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_DEPTHWISE_CONV_2D: { + return ParseDepthwiseConv2D(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_DEQUANTIZE: { + return ParseDequantize(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_FLOOR: { + return ParseFloor(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_FULLY_CONNECTED: { + return ParseFullyConnected(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_GREATER: { + return ParseGreater(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_GREATER_EQUAL: { + return ParseGreaterEqual(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_HARD_SWISH: { + return ParseHardSwish(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_L2_NORMALIZATION: { + return ParseL2Normalization(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_L2_POOL_2D: { + return ParsePool(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_LESS: { + return ParseLess(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_LESS_EQUAL: { + return ParseLessEqual(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_LOG: { + return ParseLog(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_LOGICAL_AND: { + return ParseLogicalAnd(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_LOGICAL_NOT: { + return ParseLogicalNot(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_LOGICAL_OR: { + return ParseLogicalOr(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_LOGISTIC: { + return ParseLogistic(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_MAXIMUM: { + return ParseMaximum(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_MAX_POOL_2D: { + return ParsePool(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_MEAN: { + return ParseReducer(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_MINIMUM: { + return ParseMinimum(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_MUL: { + return ParseMul(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_NEG: { + return ParseNeg(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_NOT_EQUAL: { + return ParseNotEqual(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_PACK: { + return ParsePack(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_PAD: { + return ParsePad(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_PADV2: { + return ParsePadV2(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_PRELU: { + return ParsePrelu(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_QUANTIZE: { + return ParseQuantize(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_REDUCE_ANY: { + return ParseReducer(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_REDUCE_MAX: { + return ParseReducer(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_REDUCE_MIN: { + return ParseReducer(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_REDUCE_PROD: { + return ParseReducer(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_RELU: { + return ParseRelu(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_RELU6: { + return ParseRelu6(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_RESHAPE: { + return ParseReshape(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_RESIZE_BILINEAR: { + return ParseResizeBilinear(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_RESIZE_NEAREST_NEIGHBOR: { + return ParseResizeNearestNeighbor(op, error_reporter, allocator, + builtin_data); + } + + case BuiltinOperator_ROUND: { + return ParseRound(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_RSQRT: { + return ParseRsqrt(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SHAPE: { + return ParseShape(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SIN: { + return ParseSin(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SOFTMAX: { + return ParseSoftmax(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SPLIT: { + return ParseSplit(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SPLIT_V: { + return ParseSplitV(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SQRT: { + return ParseSqrt(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SQUARE: { + return ParseSquare(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_STRIDED_SLICE: { + return ParseStridedSlice(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SUB: { + return ParseSub(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SUM: { + return ParseReducer(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_SVDF: { + return ParseSvdf(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_TANH: { + return ParseTanh(op, error_reporter, allocator, builtin_data); + } + + case BuiltinOperator_UNPACK: { + return ParseUnpack(op, error_reporter, allocator, builtin_data); + } + case BuiltinOperator_CAST: { auto params = safe_allocator.Allocate(); TF_LITE_ENSURE(error_reporter, params != nullptr); @@ -231,61 +426,13 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = params.release(); return kTfLiteOk; } - case BuiltinOperator_AVERAGE_POOL_2D: - case BuiltinOperator_MAX_POOL_2D: - case BuiltinOperator_L2_POOL_2D: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* pool_params = op->builtin_options_as_Pool2DOptions()) { - params->padding = parse_padding(pool_params->padding()); - params->stride_width = pool_params->stride_w(); - params->stride_height = pool_params->stride_h(); - params->filter_width = pool_params->filter_width(); - params->filter_height = pool_params->filter_height(); - params->activation = - parse_activation(pool_params->fused_activation_function()); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_DEPTHWISE_CONV_2D: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* conv_params = - op->builtin_options_as_DepthwiseConv2DOptions()) { - params->padding = parse_padding(conv_params->padding()); - params->stride_width = conv_params->stride_w(); - params->stride_height = conv_params->stride_h(); - params->depth_multiplier = conv_params->depth_multiplier(); - params->activation = - parse_activation(conv_params->fused_activation_function()); - - params->dilation_width_factor = conv_params->dilation_w_factor(); - params->dilation_height_factor = conv_params->dilation_h_factor(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_SVDF: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* svdf_params = op->builtin_options_as_SVDFOptions()) { - params->rank = svdf_params->rank(); - params->activation = - parse_activation(svdf_params->fused_activation_function()); - params->asymmetric_quantize_inputs = - svdf_params->asymmetric_quantize_inputs(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } case BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_RNN: { auto params = safe_allocator.Allocate(); TF_LITE_ENSURE(error_reporter, params != nullptr); if (const auto* sequence_rnn_params = op->builtin_options_as_SequenceRNNOptions()) { params->activation = - parse_activation(sequence_rnn_params->fused_activation_function()); + ConvertActivation(sequence_rnn_params->fused_activation_function()); params->time_major = sequence_rnn_params->time_major(); params->asymmetric_quantize_inputs = sequence_rnn_params->asymmetric_quantize_inputs(); @@ -299,7 +446,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, TF_LITE_ENSURE(error_reporter, params != nullptr); if (const auto* bidi_sequence_rnn_params = op->builtin_options_as_BidirectionalSequenceRNNOptions()) { - params->activation = parse_activation( + params->activation = ConvertActivation( bidi_sequence_rnn_params->fused_activation_function()); params->time_major = bidi_sequence_rnn_params->time_major(); params->merge_outputs = bidi_sequence_rnn_params->merge_outputs(); @@ -314,7 +461,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, TF_LITE_ENSURE(error_reporter, params != nullptr); if (const auto* rnn_params = op->builtin_options_as_RNNOptions()) { params->activation = - parse_activation(rnn_params->fused_activation_function()); + ConvertActivation(rnn_params->fused_activation_function()); params->asymmetric_quantize_inputs = rnn_params->asymmetric_quantize_inputs(); } @@ -332,104 +479,16 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = params.release(); return kTfLiteOk; } - case BuiltinOperator_FULLY_CONNECTED: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* fully_connected_params = - op->builtin_options_as_FullyConnectedOptions()) { - params->activation = parse_activation( - fully_connected_params->fused_activation_function()); - params->keep_num_dims = fully_connected_params->keep_num_dims(); - params->asymmetric_quantize_inputs = - fully_connected_params->asymmetric_quantize_inputs(); - switch (fully_connected_params->weights_format()) { - case FullyConnectedOptionsWeightsFormat_DEFAULT: - params->weights_format = kTfLiteFullyConnectedWeightsFormatDefault; - break; - case FullyConnectedOptionsWeightsFormat_SHUFFLED4x16INT8: - params->weights_format = - kTfLiteFullyConnectedWeightsFormatShuffled4x16Int8; - break; - default: - TF_LITE_REPORT_ERROR(error_reporter, - "Unhandled fully-connected weights format."); - return kTfLiteError; - } - } - *builtin_data = params.release(); - return kTfLiteOk; - } + case BuiltinOperator_HASHTABLE_LOOKUP: // no-op. return kTfLiteOk; - case BuiltinOperator_SOFTMAX: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* softmax_params = - op->builtin_options_as_SoftmaxOptions()) { - params->beta = softmax_params->beta(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_CONCATENATION: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* concatenation_params = - op->builtin_options_as_ConcatenationOptions()) { - params->activation = - parse_activation(concatenation_params->fused_activation_function()); - params->axis = concatenation_params->axis(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_MUL: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_MulOptions()) { - params->activation = - parse_activation(schema_params->fused_activation_function()); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_ADD: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_AddOptions()) { - params->activation = - parse_activation(schema_params->fused_activation_function()); - } - *builtin_data = params.release(); - return kTfLiteOk; - } case BuiltinOperator_DIV: { auto params = safe_allocator.Allocate(); TF_LITE_ENSURE(error_reporter, params != nullptr); if (const auto* schema_params = op->builtin_options_as_DivOptions()) { params->activation = - parse_activation(schema_params->fused_activation_function()); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_SUB: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_SubOptions()) { - params->activation = - parse_activation(schema_params->fused_activation_function()); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_L2_NORMALIZATION: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_L2NormOptions()) { - params->activation = - parse_activation(schema_params->fused_activation_function()); + ConvertActivation(schema_params->fused_activation_function()); } *builtin_data = params.release(); return kTfLiteOk; @@ -452,7 +511,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, TF_LITE_ENSURE(error_reporter, params != nullptr); if (const auto* lstm_params = op->builtin_options_as_LSTMOptions()) { params->activation = - parse_activation(lstm_params->fused_activation_function()); + ConvertActivation(lstm_params->fused_activation_function()); params->cell_clip = lstm_params->cell_clip(); params->proj_clip = lstm_params->proj_clip(); switch (lstm_params->kernel_type()) { @@ -485,7 +544,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, if (const auto* seq_lstm_params = op->builtin_options_as_UnidirectionalSequenceLSTMOptions()) { params->activation = - parse_activation(seq_lstm_params->fused_activation_function()); + ConvertActivation(seq_lstm_params->fused_activation_function()); params->cell_clip = seq_lstm_params->cell_clip(); params->proj_clip = seq_lstm_params->proj_clip(); params->time_major = seq_lstm_params->time_major(); @@ -502,7 +561,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, if (const auto* bidi_lstm_params = op->builtin_options_as_BidirectionalSequenceLSTMOptions()) { params->activation = - parse_activation(bidi_lstm_params->fused_activation_function()); + ConvertActivation(bidi_lstm_params->fused_activation_function()); params->cell_clip = bidi_lstm_params->cell_clip(); params->proj_clip = bidi_lstm_params->proj_clip(); params->merge_outputs = bidi_lstm_params->merge_outputs(); @@ -513,51 +572,6 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = params.release(); return kTfLiteOk; } - case BuiltinOperator_RESIZE_BILINEAR: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = - op->builtin_options_as_ResizeBilinearOptions()) { - params->align_corners = schema_params->align_corners(); - params->half_pixel_centers = schema_params->half_pixel_centers(); - } else { - // Some older models did not populate the ResizeBilinearOptions field in - // the flatbuffer, so ensure it's set to a sensible default. - params->align_corners = false; - params->half_pixel_centers = false; - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_RESIZE_NEAREST_NEIGHBOR: { - auto params = - safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = - op->builtin_options_as_ResizeNearestNeighborOptions()) { - params->align_corners = schema_params->align_corners(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_RESHAPE: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_ReshapeOptions()) { - auto* new_shape = schema_params->new_shape(); - // TODO(b/147203660): We need to figure out when dynamic reshape - // (new_shape is a tensor) happens, why the option is not a nullptr. - // But nonethless, we should only copy when new_shape is not a nullptr. - if (new_shape) { - TF_LITE_ENSURE_STATUS(FlatBufferIntVectorToArray( - sizeof(params->shape), new_shape, params->shape, error_reporter, - "reshape")); - params->num_dimensions = new_shape->size(); - } - } - *builtin_data = params.release(); - return kTfLiteOk; - } case BuiltinOperator_SKIP_GRAM: { auto params = safe_allocator.Allocate(); TF_LITE_ENSURE(error_reporter, params != nullptr); @@ -601,83 +615,20 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = params.release(); return kTfLiteOk; } - case BuiltinOperator_MEAN: - case BuiltinOperator_REDUCE_MAX: - case BuiltinOperator_REDUCE_MIN: - case BuiltinOperator_REDUCE_PROD: - case BuiltinOperator_REDUCE_ANY: - case BuiltinOperator_SUM: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_ReducerOptions()) { - params->keep_dims = schema_params->keep_dims(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_SPLIT: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_SplitOptions()) { - params->num_splits = schema_params->num_splits(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_SPLIT_V: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_SplitVOptions()) { - params->num_splits = schema_params->num_splits(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } + case BuiltinOperator_SQUEEZE: { auto params = safe_allocator.Allocate(); TF_LITE_ENSURE(error_reporter, params != nullptr); if (const auto* schema_params = op->builtin_options_as_SqueezeOptions()) { const auto* squeeze_dims = schema_params->squeeze_dims(); - TF_LITE_ENSURE_STATUS(FlatBufferIntVectorToArray( - sizeof(params->squeeze_dims), squeeze_dims, params->squeeze_dims, - error_reporter, "squeeze")); - params->num_squeeze_dims = squeeze_dims->size(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_STRIDED_SLICE: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = - op->builtin_options_as_StridedSliceOptions()) { - params->begin_mask = schema_params->begin_mask(); - params->end_mask = schema_params->end_mask(); - params->ellipsis_mask = schema_params->ellipsis_mask(); - params->new_axis_mask = schema_params->new_axis_mask(); - params->shrink_axis_mask = schema_params->shrink_axis_mask(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_ARG_MAX: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_ArgMaxOptions()) { - TF_LITE_ENSURE_STATUS(ConvertTensorType(schema_params->output_type(), - ¶ms->output_type, - error_reporter)); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_ARG_MIN: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_ArgMinOptions()) { - TF_LITE_ENSURE_STATUS(ConvertTensorType(schema_params->output_type(), - ¶ms->output_type, - error_reporter)); + if (squeeze_dims != nullptr) { + TF_LITE_ENSURE_STATUS(FlatBufferIntVectorToArray( + sizeof(params->squeeze_dims), squeeze_dims, params->squeeze_dims, + error_reporter, "squeeze")); + params->num_squeeze_dims = squeeze_dims->size(); + } else { + params->num_squeeze_dims = 0; + } } *builtin_data = params.release(); return kTfLiteOk; @@ -687,7 +638,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, TF_LITE_ENSURE(error_reporter, params != nullptr); if (const auto* transpose_conv_params = op->builtin_options_as_TransposeConvOptions()) { - params->padding = parse_padding(transpose_conv_params->padding()); + params->padding = ConvertPadding(transpose_conv_params->padding()); params->stride_width = transpose_conv_params->stride_w(); params->stride_height = transpose_conv_params->stride_h(); } @@ -704,26 +655,6 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = params.release(); return kTfLiteOk; } - case BuiltinOperator_SHAPE: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* schema_params = op->builtin_options_as_ShapeOptions()) { - TF_LITE_ENSURE_STATUS(ConvertTensorType( - schema_params->out_type(), ¶ms->out_type, error_reporter)); - } - *builtin_data = params.release(); - return kTfLiteOk; - } - case BuiltinOperator_PACK: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* pack_params = op->builtin_options_as_PackOptions()) { - params->values_count = pack_params->values_count(); - params->axis = pack_params->axis(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } case BuiltinOperator_DELEGATE: { // TODO(ycling): Revisit when supporting saving delegated models. TF_LITE_REPORT_ERROR(error_reporter, @@ -752,16 +683,6 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = params.release(); return kTfLiteOk; } - case BuiltinOperator_UNPACK: { - auto params = safe_allocator.Allocate(); - TF_LITE_ENSURE(error_reporter, params != nullptr); - if (const auto* unpack_params = op->builtin_options_as_UnpackOptions()) { - params->num = unpack_params->num(); - params->axis = unpack_params->axis(); - } - *builtin_data = params.release(); - return kTfLiteOk; - } case BuiltinOperator_LEAKY_RELU: { auto params = safe_allocator.Allocate(); TF_LITE_ENSURE(error_reporter, params != nullptr); @@ -840,8 +761,27 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = params.release(); return kTfLiteOk; } + case BuiltinOperator_CALL_ONCE: { + auto params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + if (const auto* call_once_params = + op->builtin_options_as_CallOnceOptions()) { + params->init_subgraph_index = call_once_params->init_subgraph_index(); + } + *builtin_data = params.release(); + return kTfLiteOk; + } + case BuiltinOperator_CUMSUM: { + auto params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + if (const auto* cumsum_params = op->builtin_options_as_CumsumOptions()) { + params->exclusive = cumsum_params->exclusive(); + params->reverse = cumsum_params->reverse(); + } + *builtin_data = params.release(); + return kTfLiteOk; + } // Below are the ops with no builtin_data structure. - case BuiltinOperator_ABS: case BuiltinOperator_BATCH_TO_SPACE_ND: // TODO(aselle): Implement call in BuiltinOptions, but nullptrs are // ok for now, since there is no call implementation either. @@ -849,52 +789,24 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, case BuiltinOperator_CONCAT_EMBEDDINGS: case BuiltinOperator_COS: case BuiltinOperator_CUSTOM: - case BuiltinOperator_DEQUANTIZE: case BuiltinOperator_ELU: case BuiltinOperator_EMBEDDING_LOOKUP: case BuiltinOperator_EQUAL: case BuiltinOperator_EXP: case BuiltinOperator_EXPAND_DIMS: - case BuiltinOperator_CEIL: - case BuiltinOperator_FLOOR: - case BuiltinOperator_GREATER: - case BuiltinOperator_GREATER_EQUAL: - case BuiltinOperator_HARD_SWISH: - case BuiltinOperator_LESS: - case BuiltinOperator_LESS_EQUAL: - case BuiltinOperator_LOG: - case BuiltinOperator_LOGISTIC: case BuiltinOperator_LOG_SOFTMAX: case BuiltinOperator_MATRIX_DIAG: case BuiltinOperator_MATRIX_SET_DIAG: - case BuiltinOperator_MAXIMUM: - case BuiltinOperator_MINIMUM: - case BuiltinOperator_NEG: - case BuiltinOperator_NOT_EQUAL: - case BuiltinOperator_PAD: - case BuiltinOperator_PADV2: - case BuiltinOperator_PRELU: - case BuiltinOperator_RELU: - case BuiltinOperator_RELU6: case BuiltinOperator_RELU_N1_TO_1: - case BuiltinOperator_ROUND: - case BuiltinOperator_RSQRT: case BuiltinOperator_SELECT: case BuiltinOperator_SELECT_V2: - case BuiltinOperator_SIN: case BuiltinOperator_SLICE: case BuiltinOperator_SPACE_TO_BATCH_ND: - case BuiltinOperator_SQRT: - case BuiltinOperator_TANH: case BuiltinOperator_TILE: case BuiltinOperator_TOPK_V2: case BuiltinOperator_TRANSPOSE: case BuiltinOperator_POW: - case BuiltinOperator_LOGICAL_OR: - case BuiltinOperator_LOGICAL_AND: - case BuiltinOperator_LOGICAL_NOT: case BuiltinOperator_FLOOR_DIV: - case BuiltinOperator_SQUARE: case BuiltinOperator_ZEROS_LIKE: case BuiltinOperator_FILL: case BuiltinOperator_FLOOR_MOD: @@ -905,15 +817,999 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, case BuiltinOperator_GATHER_ND: case BuiltinOperator_WHERE: case BuiltinOperator_RANK: - case BuiltinOperator_QUANTIZE: case BuiltinOperator_NON_MAX_SUPPRESSION_V4: case BuiltinOperator_NON_MAX_SUPPRESSION_V5: case BuiltinOperator_SCATTER_ND: case BuiltinOperator_DENSIFY: case BuiltinOperator_SEGMENT_SUM: + case BuiltinOperator_BROADCAST_TO: return kTfLiteOk; + case BuiltinOperator_PLACEHOLDER_FOR_GREATER_OP_CODES: + return kTfLiteError; } return kTfLiteError; } // NOLINT[readability/fn_size] +#endif // !defined(TF_LITE_STATIC_MEMORY) +} // namespace + +TfLiteStatus ConvertTensorType(TensorType tensor_type, TfLiteType* type, + ErrorReporter* error_reporter) { + switch (tensor_type) { + case TensorType_FLOAT16: + *type = kTfLiteFloat16; + return kTfLiteOk; + case TensorType_FLOAT32: + *type = kTfLiteFloat32; + return kTfLiteOk; + case TensorType_FLOAT64: + *type = kTfLiteFloat64; + return kTfLiteOk; + case TensorType_INT16: + *type = kTfLiteInt16; + return kTfLiteOk; + case TensorType_INT32: + *type = kTfLiteInt32; + return kTfLiteOk; + case TensorType_UINT8: + *type = kTfLiteUInt8; + return kTfLiteOk; + case TensorType_INT8: + *type = kTfLiteInt8; + return kTfLiteOk; + case TensorType_INT64: + *type = kTfLiteInt64; + return kTfLiteOk; + case TensorType_STRING: + *type = kTfLiteString; + return kTfLiteOk; + case TensorType_BOOL: + *type = kTfLiteBool; + return kTfLiteOk; + case TensorType_COMPLEX64: + *type = kTfLiteComplex64; + return kTfLiteOk; + case TensorType_COMPLEX128: + *type = kTfLiteComplex128; + return kTfLiteOk; + default: + *type = kTfLiteNoType; + TF_LITE_REPORT_ERROR(error_reporter, + "Unsupported data type %d in tensor\n", tensor_type); + return kTfLiteError; + } +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseAbs(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseAdd(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const AddOptions* schema_params = op->builtin_options_as_AddOptions(); + + if (schema_params != nullptr) { + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + params->pot_scale_int16 = schema_params->pot_scale_int16(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseArgMax(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const ArgMaxOptions* schema_params = op->builtin_options_as_ArgMaxOptions(); + + if (schema_params != nullptr) { + TF_LITE_ENSURE_STATUS(ConvertTensorType( + schema_params->output_type(), ¶ms->output_type, error_reporter)); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseArgMin(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const ArgMinOptions* schema_params = op->builtin_options_as_ArgMinOptions(); + + if (schema_params != nullptr) { + TF_LITE_ENSURE_STATUS(ConvertTensorType( + schema_params->output_type(), ¶ms->output_type, error_reporter)); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseCeil(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseConcatenation(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const ConcatenationOptions* schema_params = + op->builtin_options_as_ConcatenationOptions(); + + if (schema_params != nullptr) { + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + params->axis = schema_params->axis(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseConv2D(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const Conv2DOptions* schema_params = op->builtin_options_as_Conv2DOptions(); + + if (schema_params != nullptr) { + params->padding = ConvertPadding(schema_params->padding()); + params->stride_width = schema_params->stride_w(); + params->stride_height = schema_params->stride_h(); + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + + params->dilation_width_factor = schema_params->dilation_w_factor(); + params->dilation_height_factor = schema_params->dilation_h_factor(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseCos(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseDepthwiseConv2D(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const DepthwiseConv2DOptions* schema_params = + op->builtin_options_as_DepthwiseConv2DOptions(); + + if (schema_params != nullptr) { + params->padding = ConvertPadding(schema_params->padding()); + params->stride_width = schema_params->stride_w(); + params->stride_height = schema_params->stride_h(); + params->depth_multiplier = schema_params->depth_multiplier(); + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + + params->dilation_width_factor = schema_params->dilation_w_factor(); + params->dilation_height_factor = schema_params->dilation_h_factor(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseDequantize(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseEqual(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseFloor(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseFullyConnected(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const FullyConnectedOptions* schema_params = + op->builtin_options_as_FullyConnectedOptions(); + + if (schema_params != nullptr) { + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + params->keep_num_dims = schema_params->keep_num_dims(); + params->asymmetric_quantize_inputs = + schema_params->asymmetric_quantize_inputs(); + + switch (schema_params->weights_format()) { + case FullyConnectedOptionsWeightsFormat_DEFAULT: + params->weights_format = kTfLiteFullyConnectedWeightsFormatDefault; + break; + case FullyConnectedOptionsWeightsFormat_SHUFFLED4x16INT8: + params->weights_format = + kTfLiteFullyConnectedWeightsFormatShuffled4x16Int8; + break; + default: + TF_LITE_REPORT_ERROR(error_reporter, + "Unhandled fully-connected weights format."); + return kTfLiteError; + } + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseGreater(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseGreaterEqual(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseHardSwish(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseL2Normalization(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const L2NormOptions* schema_params = op->builtin_options_as_L2NormOptions(); + + if (schema_params != nullptr) { + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseLess(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseLessEqual(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseLog(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseLogicalAnd(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseLogicalNot(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseLogicalOr(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseLogistic(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseMaximum(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseMinimum(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseMul(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const MulOptions* schema_params = op->builtin_options_as_MulOptions(); + + if (schema_params != nullptr) { + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseNeg(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseNotEqual(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +TfLiteStatus ParsePack(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const PackOptions* schema_params = op->builtin_options_as_PackOptions(); + + if (schema_params != nullptr) { + params->values_count = schema_params->values_count(); + params->axis = schema_params->axis(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParsePad(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParsePadV2(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParsePool(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const Pool2DOptions* schema_params = op->builtin_options_as_Pool2DOptions(); + + if (schema_params != nullptr) { + params->padding = ConvertPadding(schema_params->padding()); + params->stride_width = schema_params->stride_w(); + params->stride_height = schema_params->stride_h(); + params->filter_width = schema_params->filter_width(); + params->filter_height = schema_params->filter_height(); + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParsePrelu(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseQuantize(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseReducer(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const ReducerOptions* schema_params = op->builtin_options_as_ReducerOptions(); + + if (schema_params != nullptr) { + params->keep_dims = schema_params->keep_dims(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseRelu(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseRelu6(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseReshape(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const ReshapeOptions* schema_params = op->builtin_options_as_ReshapeOptions(); + + if (schema_params != nullptr) { + const flatbuffers::Vector* new_shape = schema_params->new_shape(); + if (new_shape != nullptr) { + TF_LITE_ENSURE_STATUS( + FlatBufferIntVectorToArray(sizeof(params->shape), new_shape, + params->shape, error_reporter, "reshape")); + params->num_dimensions = new_shape->size(); + } else { + // TODO(b/157480169) TODO(b/147203660): We should either return + // kTfLiteError or fill in some reasonable defaults in the params struct. + // We are not doing so until we better undertand the ramifications of + // changing the legacy behavior. + } + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseResizeBilinear(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const ResizeBilinearOptions* schema_params = + op->builtin_options_as_ResizeBilinearOptions(); + + if (schema_params != nullptr) { + params->align_corners = schema_params->align_corners(); + params->half_pixel_centers = schema_params->half_pixel_centers(); + } else { + params->align_corners = false; + params->half_pixel_centers = false; + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseResizeNearestNeighbor(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const ResizeNearestNeighborOptions* schema_params = + op->builtin_options_as_ResizeNearestNeighborOptions(); + + if (schema_params != nullptr) { + params->align_corners = schema_params->align_corners(); + params->half_pixel_centers = schema_params->half_pixel_centers(); + } else { + params->align_corners = false; + params->half_pixel_centers = false; + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseRound(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseRsqrt(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseShape(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const ShapeOptions* schema_params = op->builtin_options_as_ShapeOptions(); + + if (schema_params != nullptr) { + TF_LITE_ENSURE_STATUS(ConvertTensorType(schema_params->out_type(), + ¶ms->out_type, error_reporter)); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseSin(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseSoftmax(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const SoftmaxOptions* schema_params = op->builtin_options_as_SoftmaxOptions(); + + if (schema_params != nullptr) { + params->beta = schema_params->beta(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseSplit(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const SplitOptions* schema_params = op->builtin_options_as_SplitOptions(); + + if (schema_params != nullptr) { + params->num_splits = schema_params->num_splits(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseSplitV(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + SafeBuiltinDataAllocator safe_allocator(allocator); + + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const SplitVOptions* schema_params = op->builtin_options_as_SplitVOptions(); + + if (schema_params != nullptr) { + params->num_splits = schema_params->num_splits(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseSqrt(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseSquare(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseStridedSlice(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const StridedSliceOptions* schema_params = + op->builtin_options_as_StridedSliceOptions(); + + if (schema_params != nullptr) { + params->begin_mask = schema_params->begin_mask(); + params->end_mask = schema_params->end_mask(); + params->ellipsis_mask = schema_params->ellipsis_mask(); + params->new_axis_mask = schema_params->new_axis_mask(); + params->shrink_axis_mask = schema_params->shrink_axis_mask(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseSub(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const SubOptions* schema_params = op->builtin_options_as_SubOptions(); + + if (schema_params != nullptr) { + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + params->pot_scale_int16 = schema_params->pot_scale_int16(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseSvdf(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const SVDFOptions* schema_params = op->builtin_options_as_SVDFOptions(); + if (schema_params != nullptr) { + params->rank = schema_params->rank(); + params->activation = + ConvertActivation(schema_params->fused_activation_function()); + params->asymmetric_quantize_inputs = + schema_params->asymmetric_quantize_inputs(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseTanh(const Operator*, ErrorReporter*, BuiltinDataAllocator*, + void**) { + return kTfLiteOk; +} + +TfLiteStatus ParseUnpack(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { + CheckParsePointerParams(op, error_reporter, allocator, builtin_data); + + SafeBuiltinDataAllocator safe_allocator(allocator); + std::unique_ptr + params = safe_allocator.Allocate(); + TF_LITE_ENSURE(error_reporter, params != nullptr); + + const UnpackOptions* schema_params = op->builtin_options_as_UnpackOptions(); + + if (schema_params != nullptr) { + params->num = schema_params->num(); + params->axis = schema_params->axis(); + } else { + // TODO(b/157480169): We should either return kTfLiteError or fill in some + // reasonable defaults in the params struct. We are not doing so until we + // better undertand the ramifications of changing the legacy behavior. + } + + *builtin_data = params.release(); + return kTfLiteOk; +} + +TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data) { +// TODO(b/145762662): It would be preferable to have the build graph for TF Lite +// Micro not have the ParseOpData function at all. This would require splitting +// the current file into two separate files, one of which defines the +// ParseOpData function and the other that defines the operator specific parse +// functions (e.g. ParseAdd). +// +// Such a split was attempted but was not worth the effort at the time because +// of the following reasons: +// * We could either duplicate the functions and the SafeBuiltinDataAllocator +// class in the anonymous namespace of this file, or attempt to make a common +// library with these helper functions and class. +// * Making a common library with a separate build target was not feasible as +// it introduced circular dependencies due to the ErrorReporter and a common +// .cc and .h within the same api build target the also cause circular +// dependencies due to the BuiltinDataAllocator class. +// * If all the builtin operators were to have their own parse functions, or we +// were ok with some amount of code duplication, then this split of the .cc +// files would be a lot more feasible. +#ifdef TF_LITE_STATIC_MEMORY + TF_LITE_REPORT_ERROR( + error_reporter, + "ParseOpData is unsupported on TfLiteMicro, please use the operator " + "specific parse functions (e.g. ParseAdd etc.).\n"); + return kTfLiteError; +#else + return ParseOpDataTfLite(op, op_type, error_reporter, allocator, + builtin_data); +#endif +} } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/core/api/flatbuffer_conversions.h b/code/lib/tfmicro/tensorflow/lite/core/api/flatbuffer_conversions.h index d774afe8..13680997 100644 --- a/code/lib/tfmicro/tensorflow/lite/core/api/flatbuffer_conversions.h +++ b/code/lib/tfmicro/tensorflow/lite/core/api/flatbuffer_conversions.h @@ -19,9 +19,12 @@ limitations under the License. // flatbuffer serialization format into in-memory values that are used by the // runtime API and interpreter. +#include +#include +#include + #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" -#include "tensorflow/lite/core/api/op_resolver.h" #include "tensorflow/lite/schema/schema_generated.h" namespace tflite { @@ -42,7 +45,7 @@ class BuiltinDataAllocator { // platform targets support that properly. static_assert(std::is_pod::value, "Builtin data structure must be POD."); void* allocated_memory = this->Allocate(sizeof(T), alignof(T)); - return new (allocated_memory) T; + return new (allocated_memory) T(); } virtual ~BuiltinDataAllocator() {} @@ -66,6 +69,196 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, TfLiteStatus ConvertTensorType(TensorType tensor_type, TfLiteType* type, ErrorReporter* error_reporter); +TfLiteStatus ParseAbs(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseAdd(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseArgMax(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseArgMin(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseCeil(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseConcatenation(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseConv2D(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseCos(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseDepthwiseConv2D(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseDequantize(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseEqual(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseFloor(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseFullyConnected(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseGreater(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseGreaterEqual(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseHardSwish(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseL2Normalization(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseLess(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseLessEqual(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseLog(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseLogicalAnd(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseLogicalNot(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseLogicalOr(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseLogistic(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseMaximum(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseMinimum(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseMul(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseNeg(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseNotEqual(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParsePack(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParsePad(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParsePadV2(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParsePool(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParsePrelu(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseQuantize(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseReducer(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseRelu(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseRelu6(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseReshape(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseResizeBilinear(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseResizeNearestNeighbor(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseRound(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseRsqrt(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseShape(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseSin(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseSoftmax(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseSplit(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseSplitV(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseSqrt(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseSquare(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseStridedSlice(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + +TfLiteStatus ParseSub(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseSvdf(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseTanh(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + +TfLiteStatus ParseUnpack(const Operator* op, ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, void** builtin_data); + } // namespace tflite #endif // TENSORFLOW_LITE_CORE_API_FLATBUFFER_CONVERSIONS_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/core/api/op_resolver.cc b/code/lib/tfmicro/tensorflow/lite/core/api/op_resolver.cc index 6424071f..c5dffb63 100644 --- a/code/lib/tfmicro/tensorflow/lite/core/api/op_resolver.cc +++ b/code/lib/tfmicro/tensorflow/lite/core/api/op_resolver.cc @@ -15,6 +15,11 @@ limitations under the License. #include "tensorflow/lite/core/api/op_resolver.h" +#include "flatbuffers/flatbuffers.h" // from @flatbuffers +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/schema/schema_utils.h" + namespace tflite { TfLiteStatus GetRegistrationFromOpCode( @@ -22,7 +27,7 @@ TfLiteStatus GetRegistrationFromOpCode( ErrorReporter* error_reporter, const TfLiteRegistration** registration) { TfLiteStatus status = kTfLiteOk; *registration = nullptr; - auto builtin_code = opcode->builtin_code(); + auto builtin_code = GetBuiltinCode(opcode); int version = opcode->version(); if (builtin_code > BuiltinOperator_MAX || diff --git a/code/lib/tfmicro/tensorflow/lite/core/api/op_resolver.h b/code/lib/tfmicro/tensorflow/lite/core/api/op_resolver.h index 1294b7b8..b6a8171d 100644 --- a/code/lib/tfmicro/tensorflow/lite/core/api/op_resolver.h +++ b/code/lib/tfmicro/tensorflow/lite/core/api/op_resolver.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_CORE_API_OP_RESOLVER_H_ #define TENSORFLOW_LITE_CORE_API_OP_RESOLVER_H_ +#include + #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/schema/schema_generated.h" @@ -32,6 +34,16 @@ class OpResolver { /// Finds the op registration of a custom operator by op name. virtual const TfLiteRegistration* FindOp(const char* op, int version) const = 0; + + // Returns optional delegates for resolving and handling ops in the flatbuffer + // model. This may be used in addition to the standard TfLiteRegistration + // lookup for graph resolution. + using TfLiteDelegatePtrVector = + std::vector>; + virtual TfLiteDelegatePtrVector GetDelegates(int num_threads) const { + return TfLiteDelegatePtrVector(); + } + virtual ~OpResolver() {} }; diff --git a/code/lib/tfmicro/tensorflow/lite/core/api/profiler.h b/code/lib/tfmicro/tensorflow/lite/core/api/profiler.h new file mode 100644 index 00000000..897efbe1 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/core/api/profiler.h @@ -0,0 +1,194 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_CORE_API_PROFILER_H_ +#define TENSORFLOW_LITE_CORE_API_PROFILER_H_ + +#include + +namespace tflite { + +// A simple utility for enabling profiled event tracing in TensorFlow Lite. +class Profiler { + public: + // As certain Profiler instance might be only interested in certain event + // types, we define each event type value to allow a Profiler to use + // bitmasking bitwise operations to determine whether an event should be + // recorded or not. + enum class EventType { + // Default event type, the metadata field has no special significance. + DEFAULT = 1, + + // The event is an operator invocation and the event_metadata field is the + // index of operator node. + OPERATOR_INVOKE_EVENT = 2, + + // The event is an invocation for an internal operator of a TFLite delegate. + // The event_metadata field is the index of operator node that's specific to + // the delegate. + DELEGATE_OPERATOR_INVOKE_EVENT = 4, + + // The event is a recording of runtime instrumentation such as the overall + // TFLite runtime status, the TFLite delegate status (if a delegate + // is applied), and the overall model inference latency etc. + // Note, the delegate status and overall status are stored as separate + // event_metadata fields. In particular, the delegate status is encoded + // as DelegateStatus::full_status(). + GENERAL_RUNTIME_INSTRUMENTATION_EVENT = 8, + }; + + virtual ~Profiler() {} + + // Signals the beginning of an event and returns a handle to the profile + // event. The `event_metadata1` and `event_metadata2` have different + // interpretations based on the actual Profiler instance and the `event_type`. + // For example, as for the 'SubgraphAwareProfiler' defined in + // lite/core/subgraph.h, when the event_type is OPERATOR_INVOKE_EVENT, + // `event_metadata1` represents the index of a TFLite node, and + // `event_metadata2` represents the index of the subgraph that this event + // comes from. + virtual uint32_t BeginEvent(const char* tag, EventType event_type, + int64_t event_metadata1, + int64_t event_metadata2) = 0; + // Similar w/ the above, but `event_metadata2` defaults to 0. + uint32_t BeginEvent(const char* tag, EventType event_type, + int64_t event_metadata) { + return BeginEvent(tag, event_type, event_metadata, /*event_metadata2*/ 0); + } + + // Signals an end to the specified profile event with 'event_metadata's, This + // is useful when 'event_metadata's are not available when the event begins + // or when one wants to overwrite the 'event_metadata's set at the beginning. + virtual void EndEvent(uint32_t event_handle, int64_t event_metadata1, + int64_t event_metadata2) {} + // Signals an end to the specified profile event. + virtual void EndEvent(uint32_t event_handle) = 0; + + // Appends an event of type 'event_type' with 'tag' and 'event_metadata' + // which started at 'start' and ended at 'end' + // Note: + // In cases were ProfileSimmarizer and tensorflow::StatsCalculator are used + // they assume the value is in "usec", if in any case subclasses + // didn't put usec, then the values are not meaningful. + // TODO karimnosseir: Revisit and make the function more clear. + void AddEvent(const char* tag, EventType event_type, uint64_t start, + uint64_t end, int64_t event_metadata) { + AddEvent(tag, event_type, start, end, event_metadata, + /*event_metadata2*/ 0); + } + + virtual void AddEvent(const char* tag, EventType event_type, uint64_t start, + uint64_t end, int64_t event_metadata1, + int64_t event_metadata2) {} + + protected: + friend class ScopedProfile; +}; + +// Adds a profile event to `profiler` that begins with the construction +// of the object and ends when the object goes out of scope. +// The lifetime of tag should be at least the lifetime of `profiler`. +// `profiler` may be null, in which case nothing is profiled. +class ScopedProfile { + public: + ScopedProfile(Profiler* profiler, const char* tag, + Profiler::EventType event_type = Profiler::EventType::DEFAULT, + int64_t event_metadata = 0) + : profiler_(profiler), event_handle_(0) { + if (profiler) { + event_handle_ = profiler_->BeginEvent(tag, event_type, event_metadata); + } + } + + ~ScopedProfile() { + if (profiler_) { + profiler_->EndEvent(event_handle_); + } + } + + protected: + Profiler* profiler_; + uint32_t event_handle_; +}; + +class ScopedOperatorProfile : public ScopedProfile { + public: + ScopedOperatorProfile(Profiler* profiler, const char* tag, int node_index) + : ScopedProfile(profiler, tag, Profiler::EventType::OPERATOR_INVOKE_EVENT, + static_cast(node_index)) {} +}; + +class ScopedDelegateOperatorProfile : public ScopedProfile { + public: + ScopedDelegateOperatorProfile(Profiler* profiler, const char* tag, + int node_index) + : ScopedProfile(profiler, tag, + Profiler::EventType::DELEGATE_OPERATOR_INVOKE_EVENT, + static_cast(node_index)) {} +}; + +class ScopedRuntimeInstrumentationProfile : public ScopedProfile { + public: + ScopedRuntimeInstrumentationProfile(Profiler* profiler, const char* tag) + : ScopedProfile( + profiler, tag, + Profiler::EventType::GENERAL_RUNTIME_INSTRUMENTATION_EVENT, -1) {} + + void set_runtime_status(int64_t delegate_status, int64_t interpreter_status) { + if (profiler_) { + delegate_status_ = delegate_status; + interpreter_status_ = interpreter_status; + } + } + + ~ScopedRuntimeInstrumentationProfile() { + if (profiler_) { + profiler_->EndEvent(event_handle_, delegate_status_, interpreter_status_); + } + } + + private: + int64_t delegate_status_; + int64_t interpreter_status_; +}; + +} // namespace tflite + +#define TFLITE_VARNAME_UNIQ_IMPL(name, ctr) name##ctr +#define TFLITE_VARNAME_UNIQ(name, ctr) TFLITE_VARNAME_UNIQ_IMPL(name, ctr) + +#define TFLITE_SCOPED_TAGGED_DEFAULT_PROFILE(profiler, tag) \ + tflite::ScopedProfile TFLITE_VARNAME_UNIQ(_profile_, __COUNTER__)( \ + (profiler), (tag)) + +#define TFLITE_SCOPED_TAGGED_OPERATOR_PROFILE(profiler, tag, node_index) \ + tflite::ScopedOperatorProfile TFLITE_VARNAME_UNIQ(_profile_, __COUNTER__)( \ + (profiler), (tag), (node_index)) + +#define TFLITE_SCOPED_DELEGATE_OPERATOR_PROFILE(profiler, tag, node_index) \ + tflite::ScopedDelegateOperatorProfile TFLITE_VARNAME_UNIQ( \ + _profile_, __COUNTER__)((profiler), (tag), (node_index)) + +#define TFLITE_ADD_RUNTIME_INSTRUMENTATION_EVENT( \ + profiler, tag, delegate_status, interpreter_status) \ + do { \ + if (!profiler) { \ + const auto handle = profiler->BeginEvent( \ + tag, Profiler::EventType::GENERAL_RUNTIME_INSTRUMENTATION_EVENT, \ + delegate_status, interpreter_status); \ + profiler->EndEvent(handle); \ + } \ + } while (false); + +#endif // TENSORFLOW_LITE_CORE_API_PROFILER_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/core/api/tensor_utils.cc b/code/lib/tfmicro/tensorflow/lite/core/api/tensor_utils.cc index d8d6fc46..3aac16b6 100644 --- a/code/lib/tfmicro/tensorflow/lite/core/api/tensor_utils.cc +++ b/code/lib/tfmicro/tensorflow/lite/core/api/tensor_utils.cc @@ -17,6 +17,8 @@ limitations under the License. #include +#include "tensorflow/lite/c/common.h" + namespace tflite { TfLiteStatus ResetVariableTensor(TfLiteTensor* tensor) { diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/common.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/common.h index c1db3587..662a1864 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/common.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/common.h @@ -55,9 +55,12 @@ inline void GetActivationMinMax(FusedActivationFunctionType ac, } } -inline float ActivationFunctionWithMinMax(float x, float output_activation_min, - float output_activation_max) { - return std::min(std::max(x, output_activation_min), output_activation_max); +template +inline T ActivationFunctionWithMinMax(T x, T output_activation_min, + T output_activation_max) { + using std::max; + using std::min; + return min(max(x, output_activation_min), output_activation_max); } // Legacy function, left for compatibility only. @@ -135,23 +138,24 @@ inline void BiasAndClamp(float clamp_min, float clamp_max, int bias_size, #endif } -inline int32 MultiplyByQuantizedMultiplierSmallerThanOneExp( - int32 x, int32 quantized_multiplier, int left_shift) { +inline int32_t MultiplyByQuantizedMultiplierSmallerThanOneExp( + int32_t x, int32_t quantized_multiplier, int left_shift) { using gemmlowp::RoundingDivideByPOT; using gemmlowp::SaturatingRoundingDoublingHighMul; return RoundingDivideByPOT( SaturatingRoundingDoublingHighMul(x, quantized_multiplier), -left_shift); } -inline int32 MultiplyByQuantizedMultiplierGreaterThanOne( - int32 x, int32 quantized_multiplier, int left_shift) { +inline int32_t MultiplyByQuantizedMultiplierGreaterThanOne( + int32_t x, int32_t quantized_multiplier, int left_shift) { using gemmlowp::SaturatingRoundingDoublingHighMul; return SaturatingRoundingDoublingHighMul(x * (1 << left_shift), quantized_multiplier); } -inline int32 MultiplyByQuantizedMultiplier(int32 x, int32 quantized_multiplier, - int shift) { +inline int32_t MultiplyByQuantizedMultiplier(int32_t x, + int32_t quantized_multiplier, + int shift) { using gemmlowp::RoundingDivideByPOT; using gemmlowp::SaturatingRoundingDoublingHighMul; int left_shift = shift > 0 ? shift : 0; @@ -161,16 +165,16 @@ inline int32 MultiplyByQuantizedMultiplier(int32 x, int32 quantized_multiplier, right_shift); } -inline int32 MultiplyByQuantizedMultiplier(int64_t x, - int32 quantized_multiplier, - int shift) { +inline int32_t MultiplyByQuantizedMultiplier(int64_t x, + int32_t quantized_multiplier, + int shift) { // Inputs: // - quantized_multiplier has fixed point at bit 31 // - shift is -31 to +7 (negative for right shift) // // Assumptions: The following input ranges are assumed // - quantize_scale>=0 (the usual range is (1<<30) to (1>>31)-1) - // - scaling is chosen so final scaled result fits in int32 + // - scaling is chosen so final scaled result fits in int32_t // - input x is in the range -(1<<47) <= x < (1<<47) assert(quantized_multiplier >= 0); assert(shift >= -31 && shift < 8); @@ -215,9 +219,9 @@ inline int CountLeadingSignBits(T integer_input) { using U = typename std::make_unsigned::type; return integer_input >= 0 ? CountLeadingZeros(static_cast(integer_input)) - 1 - : integer_input != std::numeric_limits::min() - ? CountLeadingZeros(2 * static_cast(-integer_input) - 1) - : 0; + : integer_input != std::numeric_limits::min() + ? CountLeadingZeros(2 * static_cast(-integer_input) - 1) + : 0; #endif } @@ -237,8 +241,12 @@ inline Integer FloorLog2(Integer n) { // generate INT16 LUT for function(), e.g., table exp(x) and 1/(1+x) used in // softmax -inline void gen_lut(const std::function& func, double min, - double max, int16_t* table, const int num) { +// func - the function to build the LUT for (e.g exp(x)) +// min,max - table limits +// table - pointer to buffer +// num - number of elements in the LUT +inline void gen_lut(double (*func)(double), double min, double max, + int16_t* table, const int num) { // size of table should equal to num + 1 // last element only for slope calculation double step = (max - min) / (num - 1); @@ -259,7 +267,35 @@ inline void gen_lut(const std::function& func, double min, std::min(std::max(TfLiteRound(func(max) * 32768.0), -32768.0), 32767.0); } -// int16 func table lookup, e.g., lookup exp() and 1/(1+x) used in softmax +// generate INT16 LUT for function(), e.g., table exp(x) and 1/(1+x) used in +// softmax +// func - the function to build the LUT for (e.g exp(x)) +// min,max - table limits +// table - pointer to buffer +// num - number of elements in the LUT +inline void gen_lut(float (*func)(float), float min, float max, int16_t* table, + const int num) { + // size of table should equal to num + 1 + // last element only for slope calculation + float step = (max - min) / (num - 1); + float half_step = step / 2.0f; + for (int i = 0; i < num - 1; i++) { + float sample_val = TfLiteRound(func(min + i * step) * 32768.0f); + float midpoint_interp_val = + TfLiteRound((func(min + (i + 1) * step) * 32768.0f + + TfLiteRound(func(min + i * step) * 32768.0f)) / + 2.0f); + float midpoint_val = + TfLiteRound(func(min + i * step + half_step) * 32768.0f); + float midpoint_err = midpoint_interp_val - midpoint_val; + float bias = TfLiteRound(midpoint_err / 2.0f); + table[i] = std::min(std::max(sample_val - bias, -32768.0f), 32767.0f); + } + table[num - 1] = std::min( + std::max(TfLiteRound(func(max) * 32768.0f), -32768.0f), 32767.0f); +} + +// int16_t func table lookup, e.g., lookup exp() and 1/(1+x) used in softmax inline int16_t generic_int16_table_lookup(int16_t value, const int16_t* lut) { // 512 base value, lut[513] only for calculate slope uint16_t index = static_cast(256 + (value >> 7)); @@ -410,6 +446,23 @@ SaturatingRoundingMultiplyByPOTParam( SaturatingRoundingMultiplyByPOTParam(a.raw(), exponent)); } +// Convert int32_t multiplier to int16_t with rounding. +inline void DownScaleInt32ToInt16Multiplier(int32_t multiplier_int32_t, + int16_t* multiplier_int16_t) { + TFLITE_DCHECK_GE(multiplier_int32_t, 0); + static constexpr int32_t kRoundingOffset = 1 << 15; + if (multiplier_int32_t >= + std::numeric_limits::max() - kRoundingOffset) { + *multiplier_int16_t = std::numeric_limits::max(); + return; + } + const int32_t result = (multiplier_int32_t + kRoundingOffset) >> 16; + TFLITE_DCHECK_LE(result << 16, multiplier_int32_t + kRoundingOffset); + TFLITE_DCHECK_GT(result << 16, multiplier_int32_t - kRoundingOffset); + *multiplier_int16_t = result; + TFLITE_DCHECK_EQ(*multiplier_int16_t, result); +} + // Minimum output bits to accommodate log of maximum input range. It actually // does not matter if one considers, say, [-64,64] or [-64,64). // @@ -418,15 +471,13 @@ SaturatingRoundingMultiplyByPOTParam( // ceil(log(abs( log(2.^(0:127))+1 ))/log(2)); ... // ceil(log(abs( log(2.^(0:127))+1 ))/log(2))] constexpr int min_log_x_output_bits(int input_bits) { - return input_bits > 90 - ? 7 - : input_bits > 44 - ? 6 - : input_bits > 21 - ? 5 - : input_bits > 10 - ? 4 - : input_bits > 4 ? 3 : input_bits > 1 ? 2 : 1; + return input_bits > 90 ? 7 + : input_bits > 44 ? 6 + : input_bits > 21 ? 5 + : input_bits > 10 ? 4 + : input_bits > 4 ? 3 + : input_bits > 1 ? 2 + : 1; } // Although currently the name of this function says that it cannot handle @@ -434,17 +485,17 @@ constexpr int min_log_x_output_bits(int input_bits) { // x_max is the largest representable input. In other words, the output range // is symmetric. template -inline gemmlowp::FixedPoint +inline gemmlowp::FixedPoint log_x_for_x_greater_than_or_equal_to_1_impl( - gemmlowp::FixedPoint input_val) { - // assert(__builtin_clz(0u) >= std::numeric_limits::digits - 1); - // assert(__builtin_clz(0u) <= std::numeric_limits::digits); - using FixedPoint0 = gemmlowp::FixedPoint; + gemmlowp::FixedPoint input_val) { + // assert(__builtin_clz(0u) >= std::numeric_limits::digits - 1); + // assert(__builtin_clz(0u) <= std::numeric_limits::digits); + using FixedPoint0 = gemmlowp::FixedPoint; // The reason for accumulating the result with an extra bit of headroom is // that z_pow_2_adj * log_2 might be saturated, and adding num_scaled * // recip_denom will otherwise introduce an error. static constexpr int kAccumIntegerBits = OutputIntegerBits + 1; - using FixedPointAccum = gemmlowp::FixedPoint; + using FixedPointAccum = gemmlowp::FixedPoint; const FixedPoint0 log_2 = GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT( FixedPoint0, 1488522236, std::log(2.0)); @@ -472,10 +523,10 @@ log_x_for_x_greater_than_or_equal_to_1_impl( // required shift "ourselves" instead of using, say, Rescale. FixedPoint0 z_a = FixedPoint0::FromRaw(input_val.raw()); // z_a_pow_2 = input_integer_bits - z_a_headroom; - int z_a_headroom_plus_1 = CountLeadingZeros(static_cast(z_a.raw())); + int z_a_headroom_plus_1 = CountLeadingZeros(static_cast(z_a.raw())); FixedPoint0 r_a_tmp = SaturatingRoundingMultiplyByPOTParam(z_a, (z_a_headroom_plus_1 - 1)); - const int32 r_a_raw = + const int32_t r_a_raw = SaturatingRoundingMultiplyByPOTParam((r_a_tmp * sqrt_half).raw(), 1); // z_pow_2_adj = max(z_pow_2_a - 0.75, z_pow_2_b - 0.25); // z_pow_2_adj = max(InputIntegerBits - z_a_headroom_plus_1 + 0.25, @@ -487,8 +538,8 @@ log_x_for_x_greater_than_or_equal_to_1_impl( // z_b is treated like z_a, but premultiplying by sqrt(0.5). FixedPoint0 z_b = z_a * sqrt_half; - int z_b_headroom = CountLeadingZeros(static_cast(z_b.raw())) - 1; - const int32 r_b_raw = + int z_b_headroom = CountLeadingZeros(static_cast(z_b.raw())) - 1; + const int32_t r_b_raw = SaturatingRoundingMultiplyByPOTParam(z_a.raw(), z_b_headroom); const FixedPointAccum z_b_pow_2_adj = SaturatingSub( FixedPointAccum::FromRaw(SaturatingRoundingMultiplyByPOTParam( @@ -516,9 +567,9 @@ log_x_for_x_greater_than_or_equal_to_1_impl( } template -inline gemmlowp::FixedPoint +inline gemmlowp::FixedPoint log_x_for_x_greater_than_or_equal_to_1( - gemmlowp::FixedPoint input_val) { + gemmlowp::FixedPoint input_val) { static_assert( OutputIntegerBits >= min_log_x_output_bits(InputIntegerBits), "Output integer bits must be sufficient to accommodate logs of inputs."); @@ -527,25 +578,25 @@ log_x_for_x_greater_than_or_equal_to_1( input_val); } -inline int32 GetReciprocal(int32 x, int x_integer_digits, - int* num_bits_over_unit) { - int headroom_plus_one = CountLeadingZeros(static_cast(x)); +inline int32_t GetReciprocal(int32_t x, int x_integer_digits, + int* num_bits_over_unit) { + int headroom_plus_one = CountLeadingZeros(static_cast(x)); // This is the number of bits to the left of the binary point above 1.0. // Consider x=1.25. In that case shifted_scale=0.8 and // no later adjustment will be needed. *num_bits_over_unit = x_integer_digits - headroom_plus_one; - const int32 shifted_sum_minus_one = - static_cast((static_cast(x) << headroom_plus_one) - - (static_cast(1) << 31)); + const int32_t shifted_sum_minus_one = + static_cast((static_cast(x) << headroom_plus_one) - + (static_cast(1) << 31)); - gemmlowp::FixedPoint shifted_scale = + gemmlowp::FixedPoint shifted_scale = gemmlowp::one_over_one_plus_x_for_x_in_0_1( - gemmlowp::FixedPoint::FromRaw(shifted_sum_minus_one)); + gemmlowp::FixedPoint::FromRaw(shifted_sum_minus_one)); return shifted_scale.raw(); } -inline void GetInvSqrtQuantizedMultiplierExp(int32 input, int reverse_shift, - int32* output_inv_sqrt, +inline void GetInvSqrtQuantizedMultiplierExp(int32_t input, int reverse_shift, + int32_t* output_inv_sqrt, int* output_shift) { TFLITE_DCHECK_GE(input, 0); if (input <= 1) { @@ -565,7 +616,7 @@ inline void GetInvSqrtQuantizedMultiplierExp(int32 input, int reverse_shift, ++*output_shift; } const unsigned max_left_shift_bits = - CountLeadingZeros(static_cast(input)) - 1; + CountLeadingZeros(static_cast(input)) - 1; const unsigned max_left_shift_bit_pairs = max_left_shift_bits / 2; const unsigned left_shift_bit_pairs = max_left_shift_bit_pairs - 1; *output_shift -= left_shift_bit_pairs; @@ -577,8 +628,8 @@ inline void GetInvSqrtQuantizedMultiplierExp(int32 input, int reverse_shift, using gemmlowp::SaturatingRoundingMultiplyByPOT; // Using 3 integer bits gives us enough room for the internal arithmetic in // this Newton-Raphson iteration. - using F3 = FixedPoint; - using F0 = FixedPoint; + using F3 = FixedPoint; + using F0 = FixedPoint; const F3 fixedpoint_input = F3::FromRaw(input >> 1); const F3 fixedpoint_half_input = SaturatingRoundingMultiplyByPOT<-1>(fixedpoint_input); @@ -645,6 +696,13 @@ inline int SubscriptToIndex(const NdArrayDesc<5>& desc, int indexes[5]) { indexes[4] * desc.strides[4]; } +inline int SubscriptToIndex(const NdArrayDesc<8>& desc, int indexes[8]) { + return indexes[0] * desc.strides[0] + indexes[1] * desc.strides[1] + + indexes[2] * desc.strides[2] + indexes[3] * desc.strides[3] + + indexes[4] * desc.strides[4] + indexes[5] * desc.strides[5] + + indexes[6] * desc.strides[6] + indexes[7] * desc.strides[7]; +} + // Given the dimensions of the operands for an element-wise binary broadcast, // adjusts them so that they can be directly iterated over with simple loops. // Returns the adjusted dims as instances of NdArrayDesc in 'desc0_out' and diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/compatibility.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/compatibility.h index bfd021ac..61becad3 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/compatibility.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/compatibility.h @@ -76,13 +76,15 @@ limitations under the License. #define TFLITE_CHECK_LT(x, y) ((x) < (y)) ? (void)0 : TFLITE_ABORT #endif -// TODO(ahentz): Clean up. +#ifndef TF_LITE_STATIC_MEMORY +// TODO(b/162019032): Consider removing these type-aliases. using int8 = std::int8_t; using uint8 = std::uint8_t; using int16 = std::int16_t; using uint16 = std::uint16_t; using int32 = std::int32_t; using uint32 = std::uint32_t; +#endif // !defined(TF_LITE_STATIC_MEMORY) // TFLITE_DEPRECATED() // diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/cppmath.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/cppmath.h index 611a8d25..24a3aec8 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/cppmath.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/cppmath.h @@ -19,8 +19,9 @@ limitations under the License. namespace tflite { -#if defined(TF_LITE_USE_GLOBAL_CMATH_FUNCTIONS) || \ - (defined(__ANDROID__) && !defined(__NDK_MAJOR__)) || defined(ARDUINO) +#if defined(TF_LITE_USE_GLOBAL_CMATH_FUNCTIONS) || \ + (defined(__ANDROID__) && !defined(__NDK_MAJOR__)) || defined(ARDUINO) || \ + defined(__ZEPHYR__) #define TF_LITE_GLOBAL_STD_PREFIX #else #define TF_LITE_GLOBAL_STD_PREFIX std diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_optional_debug_tools.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/max.h similarity index 55% rename from code/lib/tfmicro/tensorflow/lite/micro/micro_optional_debug_tools.h rename to code/lib/tfmicro/tensorflow/lite/kernels/internal/max.h index ae96b62a..c1810027 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_optional_debug_tools.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/max.h @@ -1,4 +1,4 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,24 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// Optional debugging functionality. For small sized binaries, these are not -// needed. -#ifndef TENSORFLOW_LITE_MICRO_MICRO_OPTIONAL_DEBUG_TOOLS_H_ -#define TENSORFLOW_LITE_MICRO_MICRO_OPTIONAL_DEBUG_TOOLS_H_ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_MAX_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_MAX_H_ -#include "tensorflow/lite/micro/micro_interpreter.h" +#include namespace tflite { -// Prints a dump of what tensors and what nodes are in the interpreter. -void PrintInterpreterState(MicroInterpreter* interpreter); + +#if defined(TF_LITE_USE_GLOBAL_MAX) || defined(__ZEPHYR__) +inline float TfLiteMax(const float& x, const float& y) { + return std::max(x, y); +} +#else +template +inline T TfLiteMax(const T& x, const T& y) { + return std::fmax(x, y); +} +#endif + } // namespace tflite -#endif // TENSORFLOW_LITE_MICRO_MICRO_OPTIONAL_DEBUG_TOOLS_H_ +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_MAX_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/min.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/min.h new file mode 100644 index 00000000..62035dcc --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/min.h @@ -0,0 +1,35 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_MIN_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_MIN_H_ + +#include + +namespace tflite { + +#if defined(TF_LITE_USE_GLOBAL_MIN) || defined(__ZEPHYR__) +inline float TfLiteMin(const float& x, const float& y) { + return std::min(x, y); +} +#else +template +inline T TfLiteMin(const T& x, const T& y) { + return std::fmin(x, y); +} +#endif + +} // namespace tflite + +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_MIN_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/tensor.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/portable_tensor.h similarity index 80% rename from code/lib/tfmicro/tensorflow/lite/kernels/internal/tensor.h rename to code/lib/tfmicro/tensorflow/lite/kernels/internal/portable_tensor.h index 0005bf38..8b0f6d1e 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/tensor.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/portable_tensor.h @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_TENSOR_H_ -#define TENSORFLOW_LITE_KERNELS_INTERNAL_TENSOR_H_ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_PORTABLE_TENSOR_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_PORTABLE_TENSOR_H_ #include #include @@ -21,7 +21,6 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/internal/types.h" -#include "tensorflow/lite/string_util.h" namespace tflite { @@ -76,12 +75,12 @@ class VectorOfTensors { // A list of quantized tensors in a format that can be used by kernels like // split and concatenation. -class VectorOfQuantizedTensors : public VectorOfTensors { +class VectorOfQuantizedTensors : public VectorOfTensors { public: // Build with the tensors in 'tensor_list'. VectorOfQuantizedTensors(const TfLiteContext& context, const TfLiteIntArray& tensor_list) - : VectorOfTensors(context, tensor_list) { + : VectorOfTensors(context, tensor_list) { for (int i = 0; i < tensor_list.size; ++i) { TfLiteTensor* t = &context.tensors[tensor_list.data[i]]; zero_point_.push_back(t->params.zero_point); @@ -90,10 +89,10 @@ class VectorOfQuantizedTensors : public VectorOfTensors { } const float* scale() const { return scale_.data(); } - const int32* zero_point() const { return zero_point_.data(); } + const int32_t* zero_point() const { return zero_point_.data(); } private: - std::vector zero_point_; + std::vector zero_point_; std::vector scale_; }; @@ -119,26 +118,6 @@ class SequentialTensorWriter { T* output_ptr_; }; -template <> -class SequentialTensorWriter { - public: - SequentialTensorWriter(const TfLiteTensor* input, TfLiteTensor* output) - : input_(input), output_(output) {} - ~SequentialTensorWriter() { buffer_.WriteToTensor(output_, nullptr); } - - void Write(int position) { this->WriteN(position, 1); } - void WriteN(int position, int len) { - for (int i = 0; i < len; i++) { - buffer_.AddString(GetString(input_, position + i)); - } - } - - private: - const TfLiteTensor* input_; - TfLiteTensor* output_; - DynamicBuffer buffer_; -}; - } // namespace tflite -#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_TENSOR_H_ +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_PORTABLE_TENSOR_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/quantization_util.cc b/code/lib/tfmicro/tensorflow/lite/kernels/internal/quantization_util.cc index 60e30540..cf431cff 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/quantization_util.cc +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/quantization_util.cc @@ -342,13 +342,13 @@ void NudgeQuantizationRange(const float min, const float max, const float quant_max_float = static_cast(quant_max); *nudged_scale = (max - min) / (quant_max_float - quant_min_float); const float zero_point_from_min = quant_min_float - min / *nudged_scale; - uint16 nudged_zero_point; + uint16_t nudged_zero_point; if (zero_point_from_min < quant_min_float) { - nudged_zero_point = static_cast(quant_min); + nudged_zero_point = static_cast(quant_min); } else if (zero_point_from_min > quant_max_float) { - nudged_zero_point = static_cast(quant_max); + nudged_zero_point = static_cast(quant_max); } else { - nudged_zero_point = static_cast(TfLiteRound(zero_point_from_min)); + nudged_zero_point = static_cast(TfLiteRound(zero_point_from_min)); } *nudged_min = (quant_min_float - nudged_zero_point) * (*nudged_scale); *nudged_max = (quant_max_float - nudged_zero_point) * (*nudged_scale); diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/add.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/add.h index d0c40912..5be7ab4d 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/add.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/add.h @@ -51,34 +51,39 @@ inline void Add(const ArithmeticParams& params, // Element-wise add that can often be used for inner loop of broadcast add as // well as the non-broadcast add. + +// This function is used for 8-bit as well as for 16-bit, but the accumulator +// is 32-bit for both cases. The overflow does not happen due to the +// choice of the shift (20 or 15, accordingly - see add.cc for more comments). +template inline void AddElementwise(int size, const ArithmeticParams& params, - const uint8* input1_data, const uint8* input2_data, - uint8* output_data) { - TFLITE_DCHECK_GT(params.input1_offset, -256); - TFLITE_DCHECK_GT(params.input2_offset, -256); - TFLITE_DCHECK_LT(params.input1_offset, 256); - TFLITE_DCHECK_LT(params.input2_offset, 256); + const T* input1_data, const T* input2_data, + T* output_data) { + TFLITE_DCHECK_GT(params.input1_offset, -std::numeric_limits::max()); + TFLITE_DCHECK_GT(params.input2_offset, -std::numeric_limits::max()); + TFLITE_DCHECK_LT(params.input1_offset, std::numeric_limits::max()); + TFLITE_DCHECK_LT(params.input2_offset, std::numeric_limits::max()); for (int i = 0; i < size; ++i) { - const int32 input1_val = params.input1_offset + input1_data[i]; - const int32 input2_val = params.input2_offset + input2_data[i]; - const int32 shifted_input1_val = input1_val * (1 << params.left_shift); - const int32 shifted_input2_val = input2_val * (1 << params.left_shift); - const int32 scaled_input1_val = + const int32_t input1_val = params.input1_offset + input1_data[i]; + const int32_t input2_val = params.input2_offset + input2_data[i]; + const int32_t shifted_input1_val = input1_val * (1 << params.left_shift); + const int32_t shifted_input2_val = input2_val * (1 << params.left_shift); + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, params.input1_multiplier, params.input1_shift); - const int32 scaled_input2_val = + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, params.input2_multiplier, params.input2_shift); - const int32 raw_sum = scaled_input1_val + scaled_input2_val; - const int32 raw_output = + const int32_t raw_sum = scaled_input1_val + scaled_input2_val; + const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp( raw_sum, params.output_multiplier, params.output_shift) + params.output_offset; - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); - output_data[i] = static_cast(clamped_output); + output_data[i] = static_cast(clamped_output); } } @@ -86,40 +91,40 @@ inline void AddElementwise(int size, const ArithmeticParams& params, // broadcast add, so that, for example, scalar-broadcast with batch will still // be fast. inline void AddScalarBroadcast(int size, const ArithmeticParams& params, - uint8 input1_data, const uint8* input2_data, - uint8* output_data) { + uint8_t input1_data, const uint8_t* input2_data, + uint8_t* output_data) { TFLITE_DCHECK_GT(params.input1_offset, -256); TFLITE_DCHECK_GT(params.input2_offset, -256); TFLITE_DCHECK_LT(params.input1_offset, 256); TFLITE_DCHECK_LT(params.input2_offset, 256); - const int32 input1_val = params.input1_offset + input1_data; - const int32 shifted_input1_val = input1_val * (1 << params.left_shift); - const int32 scaled_input1_val = + const int32_t input1_val = params.input1_offset + input1_data; + const int32_t shifted_input1_val = input1_val * (1 << params.left_shift); + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, params.input1_multiplier, params.input1_shift); for (int i = 0; i < size; ++i) { - const int32 input2_val = params.input2_offset + input2_data[i]; - const int32 shifted_input2_val = input2_val * (1 << params.left_shift); - const int32 scaled_input2_val = + const int32_t input2_val = params.input2_offset + input2_data[i]; + const int32_t shifted_input2_val = input2_val * (1 << params.left_shift); + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, params.input2_multiplier, params.input2_shift); - const int32 raw_sum = scaled_input1_val + scaled_input2_val; - const int32 raw_output = + const int32_t raw_sum = scaled_input1_val + scaled_input2_val; + const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp( raw_sum, params.output_multiplier, params.output_shift) + params.output_offset; - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); - output_data[i] = static_cast(clamped_output); + output_data[i] = static_cast(clamped_output); } } inline void Add(const ArithmeticParams& params, - const RuntimeShape& input1_shape, const uint8* input1_data, - const RuntimeShape& input2_shape, const uint8* input2_data, - const RuntimeShape& output_shape, uint8* output_data) { + const RuntimeShape& input1_shape, const uint8_t* input1_data, + const RuntimeShape& input2_shape, const uint8_t* input2_data, + const RuntimeShape& output_shape, uint8_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); const int flat_size = @@ -132,24 +137,53 @@ inline void Add(const ArithmeticParams& params, AddElementwise(flat_size, params, input1_data, input2_data, output_data); } +inline void AddGeneralParamScale(const ArithmeticParams& params, + const RuntimeShape& input1_shape, + const int16_t* input1_data, + const RuntimeShape& input2_shape, + const int16_t* input2_data, + const RuntimeShape& output_shape, + int16_t* output_data) { + TFLITE_DCHECK_LE(params.quantized_activation_min, + params.quantized_activation_max); + const int flat_size = + MatchingElementsSize(input1_shape, input2_shape, output_shape); + + int max_value = std::numeric_limits::max(); + + TFLITE_DCHECK_GT(params.input1_offset, -max_value); + TFLITE_DCHECK_GT(params.input2_offset, -max_value); + TFLITE_DCHECK_LT(params.input1_offset, max_value); + TFLITE_DCHECK_LT(params.input2_offset, max_value); + AddElementwise(flat_size, params, input1_data, input2_data, output_data); +} + inline void Add(const ArithmeticParams& params, - const RuntimeShape& input1_shape, const int16* input1_data, - const RuntimeShape& input2_shape, const int16* input2_data, - const RuntimeShape& output_shape, int16* output_data) { + const RuntimeShape& input1_shape, const int16_t* input1_data, + const RuntimeShape& input2_shape, const int16_t* input2_data, + const RuntimeShape& output_shape, int16_t* output_data, + bool pot_scale = true) { + if (!pot_scale) { + AddGeneralParamScale(params, input1_shape, input1_data, input2_shape, + input2_data, output_shape, output_data); + return; + } + TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); const int input1_shift = params.input1_shift; const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape); - const int16 output_activation_min = params.quantized_activation_min; - const int16 output_activation_max = params.quantized_activation_max; + const int16_t output_activation_min = params.quantized_activation_min; + const int16_t output_activation_max = params.quantized_activation_max; TFLITE_DCHECK(input1_shift == 0 || params.input2_shift == 0); TFLITE_DCHECK_LE(input1_shift, 0); TFLITE_DCHECK_LE(params.input2_shift, 0); - const int16* not_shift_input = input1_shift == 0 ? input1_data : input2_data; - const int16* shift_input = input1_shift == 0 ? input2_data : input1_data; + const int16_t* not_shift_input = + input1_shift == 0 ? input1_data : input2_data; + const int16_t* shift_input = input1_shift == 0 ? input2_data : input1_data; const int input_right_shift = input1_shift == 0 ? -params.input2_shift : -input1_shift; @@ -161,8 +195,8 @@ inline void Add(const ArithmeticParams& params, F0 scaled_input = F0::FromRaw( gemmlowp::RoundingDivideByPOT(shift_input[i], input_right_shift)); F0 result = gemmlowp::SaturatingAdd(scaled_input, input_ready_scaled); - const int16 raw_output = result.raw(); - const int16 clamped_output = std::min( + const int16_t raw_output = result.raw(); + const int16_t clamped_output = std::min( output_activation_max, std::max(output_activation_min, raw_output)); output_data[i] = clamped_output; } @@ -218,11 +252,11 @@ inline void BroadcastAdd4DSlow(const ArithmeticParams& params, inline void BroadcastAdd4DSlow(const ArithmeticParams& params, const RuntimeShape& input1_shape, - const int32* input1_data, + const int32_t* input1_data, const RuntimeShape& input2_shape, - const int32* input2_data, + const int32_t* input2_data, const RuntimeShape& output_shape, - int32* output_data) { + int32_t* output_data) { NdArrayDesc<4> desc1; NdArrayDesc<4> desc2; NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1, @@ -257,13 +291,14 @@ inline void BroadcastAdd4DSlow(const ArithmeticParams& params, } } -inline void BroadcastAdd4DSlow(const ArithmeticParams& params, - const RuntimeShape& input1_shape, - const uint8* input1_data, - const RuntimeShape& input2_shape, - const uint8* input2_data, - const RuntimeShape& output_shape, - uint8* output_data) { +// This function is used for 8-bit as well as for 16-bit, but the accumulator +// is 32-bit for both cases. The overflow does not happen due to the +// choice of the shift (20 or 15, accordingly - see add.cc for more comments). +template +inline void BroadcastAdd4DSlow( + const ArithmeticParams& params, const RuntimeShape& input1_shape, + const T* input1_data, const RuntimeShape& input2_shape, + const T* input2_data, const RuntimeShape& output_shape, T* output_data) { NdArrayDesc<4> desc1; NdArrayDesc<4> desc2; NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1, @@ -286,34 +321,34 @@ inline void BroadcastAdd4DSlow(const ArithmeticParams& params, for (int y = 0; y < extended_output_shape.Dims(1); ++y) { for (int x = 0; x < extended_output_shape.Dims(2); ++x) { for (int c = 0; c < extended_output_shape.Dims(3); ++c) { - const int32 input1_val = + const int32_t input1_val = params.input1_offset + input1_data[SubscriptToIndex(desc1, b, y, x, c)]; - const int32 input2_val = + const int32_t input2_val = params.input2_offset + input2_data[SubscriptToIndex(desc2, b, y, x, c)]; - const int32 shifted_input1_val = + const int32_t shifted_input1_val = input1_val * (1 << params.left_shift); - const int32 shifted_input2_val = + const int32_t shifted_input2_val = input2_val * (1 << params.left_shift); - const int32 scaled_input1_val = + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, params.input1_multiplier, params.input1_shift); - const int32 scaled_input2_val = + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, params.input2_multiplier, params.input2_shift); - const int32 raw_sum = scaled_input1_val + scaled_input2_val; - const int32 raw_output = + const int32_t raw_sum = scaled_input1_val + scaled_input2_val; + const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp( raw_sum, params.output_multiplier, params.output_shift) + params.output_offset; - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); output_data[Offset(extended_output_shape, b, y, x, c)] = - static_cast(clamped_output); + static_cast(clamped_output); } } } @@ -322,11 +357,11 @@ inline void BroadcastAdd4DSlow(const ArithmeticParams& params, inline void BroadcastAddFivefold(const ArithmeticParams& unswitched_params, const RuntimeShape& unswitched_input1_shape, - const uint8* unswitched_input1_data, + const uint8_t* unswitched_input1_data, const RuntimeShape& unswitched_input2_shape, - const uint8* unswitched_input2_data, + const uint8_t* unswitched_input2_data, const RuntimeShape& output_shape, - uint8* output_data) { + uint8_t* output_data) { ArithmeticParams switched_params = unswitched_params; switched_params.input1_offset = unswitched_params.input2_offset; switched_params.input1_multiplier = unswitched_params.input2_multiplier; @@ -341,18 +376,18 @@ inline void BroadcastAddFivefold(const ArithmeticParams& unswitched_params, const ArithmeticParams& params = use_unswitched ? unswitched_params : switched_params; - const uint8* input1_data = + const uint8_t* input1_data = use_unswitched ? unswitched_input1_data : unswitched_input2_data; - const uint8* input2_data = + const uint8_t* input2_data = use_unswitched ? unswitched_input2_data : unswitched_input1_data; // Fivefold nested loops. The second input resets its position for each // iteration of the second loop. The first input resets its position at the // beginning of the fourth loop. The innermost loop is an elementwise add of // sections of the arrays. - uint8* output_data_ptr = output_data; - const uint8* input1_data_ptr = input1_data; - const uint8* input2_data_reset = input2_data; + uint8_t* output_data_ptr = output_data; + const uint8_t* input1_data_ptr = input1_data; + const uint8_t* input2_data_reset = input2_data; // In the fivefold pattern, y0, y2 and y4 are not broadcast, and so shared // between input shapes. y3 for input 1 is always broadcast, and so the // dimension there is 1, whereas optionally y1 might be broadcast for input 2. @@ -368,7 +403,7 @@ inline void BroadcastAddFivefold(const ArithmeticParams& unswitched_params, // General fivefold pattern, with y4 > 1 so there is a non-broadcast inner // dimension. for (int i0 = 0; i0 < y0; ++i0) { - const uint8* input2_data_ptr; + const uint8_t* input2_data_ptr; for (int i1 = 0; i1 < y1; ++i1) { input2_data_ptr = input2_data_reset; for (int i2 = 0; i2 < y2; ++i2) { @@ -397,7 +432,7 @@ inline void BroadcastAddFivefold(const ArithmeticParams& unswitched_params, // for y4 == 1 and the loop over y3 is contained within the // AddScalarBroadcast function. for (int i0 = 0; i0 < y0; ++i0) { - const uint8* input2_data_ptr; + const uint8_t* input2_data_ptr; for (int i1 = 0; i1 < y1; ++i1) { input2_data_ptr = input2_data_reset; for (int i2 = 0; i2 < y2; ++i2) { diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/comparisons.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/comparisons.h index 379a20f5..6344bdc7 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/comparisons.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/comparisons.h @@ -18,7 +18,6 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/types.h" -#include "tensorflow/lite/string_util.h" namespace tflite { @@ -51,18 +50,6 @@ inline bool LessEqualFn(T lhs, T rhs) { return lhs <= rhs; } -inline bool StringRefEqualFn(const StringRef& lhs, const StringRef& rhs) { - if (lhs.len != rhs.len) return false; - for (int i = 0; i < lhs.len; ++i) { - if (lhs.str[i] != rhs.str[i]) return false; - } - return true; -} - -inline bool StringRefNotEqualFn(const StringRef& lhs, const StringRef& rhs) { - return !StringRefEqualFn(lhs, rhs); -} - template using ComparisonFn = bool (*)(T, T); @@ -78,22 +65,6 @@ inline void ComparisonImpl( } } -template -inline void ComparisonStringImpl(const RuntimeShape& input1_shape, - const TfLiteTensor* input1, - const RuntimeShape& input2_shape, - const TfLiteTensor* input2, - const RuntimeShape& output_shape, - bool* output_data) { - const int64_t flatsize = - MatchingFlatSize(input1_shape, input2_shape, output_shape); - for (int64_t i = 0; i < flatsize; ++i) { - const auto lhs = GetString(input1, i); - const auto rhs = GetString(input2, i); - output_data[i] = F(lhs, rhs); - } -} - template F> inline void Comparison(const ComparisonParams& op_params, const RuntimeShape& input1_shape, @@ -105,30 +76,30 @@ inline void Comparison(const ComparisonParams& op_params, input2_data, output_shape, output_data); } -template F> +template F> inline void ComparisonWithScaling( const ComparisonParams& op_params, const RuntimeShape& input1_shape, const T* input1_data, const RuntimeShape& input2_shape, const T* input2_data, const RuntimeShape& output_shape, bool* output_data) { int left_shift = op_params.left_shift; - int32 input1_offset = op_params.input1_offset; - int32 input1_multiplier = op_params.input1_multiplier; + int32_t input1_offset = op_params.input1_offset; + int32_t input1_multiplier = op_params.input1_multiplier; int input1_shift = op_params.input1_shift; - int32 input2_offset = op_params.input2_offset; - int32 input2_multiplier = op_params.input2_multiplier; + int32_t input2_offset = op_params.input2_offset; + int32_t input2_multiplier = op_params.input2_multiplier; int input2_shift = op_params.input2_shift; const int64_t flatsize = MatchingFlatSize(input1_shape, input2_shape, output_shape); for (int64_t i = 0; i < flatsize; ++i) { - const int32 input1_val = input1_offset + input1_data[i]; - const int32 input2_val = input2_offset + input2_data[i]; - const int32 shifted_input1_val = input1_val * (1 << left_shift); - const int32 shifted_input2_val = input2_val * (1 << left_shift); - const int32 scaled_input1_val = + const int32_t input1_val = input1_offset + input1_data[i]; + const int32_t input2_val = input2_offset + input2_data[i]; + const int32_t shifted_input1_val = input1_val * (1 << left_shift); + const int32_t shifted_input2_val = input2_val * (1 << left_shift); + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, input1_multiplier, input1_shift); - const int32 scaled_input2_val = + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, input2_multiplier, input2_shift); output_data[i] = F(scaled_input1_val, scaled_input2_val); @@ -180,31 +151,6 @@ inline void BroadcastComparison4DSlowImpl( } } -template -inline void BroadcastComparison4DSlowStringImpl( - const RuntimeShape& unextended_input1_shape, const TfLiteTensor* input1, - const RuntimeShape& unextended_input2_shape, const TfLiteTensor* input2, - const RuntimeShape& unextended_output_shape, bool* output_data) { - const BroadcastComparison4DSlowCommon dims = - BroadcastComparison4DSlowPreprocess(unextended_input1_shape, - unextended_input2_shape, - unextended_output_shape); - - for (int b = 0; b < dims.output_shape.Dims(0); ++b) { - for (int y = 0; y < dims.output_shape.Dims(1); ++y) { - for (int x = 0; x < dims.output_shape.Dims(2); ++x) { - for (int c = 0; c < dims.output_shape.Dims(3); ++c) { - const auto lhs = - GetString(input1, SubscriptToIndex(dims.desc1, b, y, x, c)); - const auto rhs = - GetString(input2, SubscriptToIndex(dims.desc2, b, y, x, c)); - output_data[Offset(dims.output_shape, b, y, x, c)] = F(lhs, rhs); - } - } - } - } -} - template F> inline void BroadcastComparison4DSlow(const ComparisonParams& op_params, const RuntimeShape& input1_shape, @@ -218,7 +164,7 @@ inline void BroadcastComparison4DSlow(const ComparisonParams& op_params, output_shape, output_data); } -template F> +template F> inline void BroadcastComparison4DSlowWithScaling( const ComparisonParams& op_params, const RuntimeShape& unextended_input1_shape, const T* input1_data, @@ -230,29 +176,29 @@ inline void BroadcastComparison4DSlowWithScaling( unextended_output_shape); int left_shift = op_params.left_shift; - int32 input1_offset = op_params.input1_offset; - int32 input1_multiplier = op_params.input1_multiplier; + int32_t input1_offset = op_params.input1_offset; + int32_t input1_multiplier = op_params.input1_multiplier; int input1_shift = op_params.input1_shift; - int32 input2_offset = op_params.input2_offset; - int32 input2_multiplier = op_params.input2_multiplier; + int32_t input2_offset = op_params.input2_offset; + int32_t input2_multiplier = op_params.input2_multiplier; int input2_shift = op_params.input2_shift; for (int b = 0; b < dims.output_shape.Dims(0); ++b) { for (int y = 0; y < dims.output_shape.Dims(1); ++y) { for (int x = 0; x < dims.output_shape.Dims(2); ++x) { for (int c = 0; c < dims.output_shape.Dims(3); ++c) { - const int32 input1_val = + const int32_t input1_val = input1_offset + input1_data[SubscriptToIndex(dims.desc1, b, y, x, c)]; - const int32 input2_val = + const int32_t input2_val = input2_offset + input2_data[SubscriptToIndex(dims.desc2, b, y, x, c)]; - const int32 shifted_input1_val = input1_val * (1 << left_shift); - const int32 shifted_input2_val = input2_val * (1 << left_shift); - const int32 scaled_input1_val = + const int32_t shifted_input1_val = input1_val * (1 << left_shift); + const int32_t shifted_input2_val = input2_val * (1 << left_shift); + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, input1_multiplier, input1_shift); - const int32 scaled_input2_val = + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, input2_multiplier, input2_shift); output_data[Offset(dims.output_shape, b, y, x, c)] = diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/concatenation.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/concatenation.h index 958fe3ea..25959793 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/concatenation.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/concatenation.h @@ -74,14 +74,14 @@ inline void Concatenation(const ConcatenationParams& params, // when optimizng this routine further. inline void ConcatenationWithScaling(const ConcatenationParams& params, const RuntimeShape* const* input_shapes, - const uint8* const* input_data, + const uint8_t* const* input_data, const RuntimeShape& output_shape, - uint8* output_data) { + uint8_t* output_data) { int axis = params.axis; - const int32* input_zeropoint = params.input_zeropoint; + const int32_t* input_zeropoint = params.input_zeropoint; const float* input_scale = params.input_scale; int inputs_count = params.inputs_count; - const int32 output_zeropoint = params.output_zeropoint; + const int32_t output_zeropoint = params.output_zeropoint; const float output_scale = params.output_scale; const int concat_dimensions = output_shape.DimensionsCount(); @@ -110,11 +110,11 @@ inline void ConcatenationWithScaling(const ConcatenationParams& params, } const float inverse_output_scale = 1.f / output_scale; - uint8* output_ptr = output_data; + uint8_t* output_ptr = output_data; for (int k = 0; k < outer_size; k++) { for (int i = 0; i < inputs_count; ++i) { const int copy_size = input_shapes[i]->Dims(axis) * base_inner_size; - const uint8* input_ptr = input_data[i] + k * copy_size; + const uint8_t* input_ptr = input_data[i] + k * copy_size; if (input_zeropoint[i] == output_zeropoint && input_scale[i] == output_scale) { memcpy(output_ptr, input_ptr, copy_size); diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/conv.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/conv.h index 55dd869a..b912ac1b 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/conv.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/conv.h @@ -59,28 +59,31 @@ inline void Conv(const ConvParams& params, const RuntimeShape& input_shape, const int output_width = output_shape.Dims(2); for (int batch = 0; batch < batches; ++batch) { for (int out_y = 0; out_y < output_height; ++out_y) { + const int in_y_origin = (out_y * stride_height) - pad_height; for (int out_x = 0; out_x < output_width; ++out_x) { + const int in_x_origin = (out_x * stride_width) - pad_width; for (int out_channel = 0; out_channel < output_depth; ++out_channel) { - const int in_x_origin = (out_x * stride_width) - pad_width; - const int in_y_origin = (out_y * stride_height) - pad_height; float total = 0.f; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { + const int in_y = in_y_origin + dilation_height_factor * filter_y; for (int filter_x = 0; filter_x < filter_width; ++filter_x) { + const int in_x = in_x_origin + dilation_width_factor * filter_x; + + // Zero padding by omitting the areas outside the image. + const bool is_point_inside_image = + (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && + (in_y < input_height); + + if (!is_point_inside_image) { + continue; + } + for (int in_channel = 0; in_channel < input_depth; ++in_channel) { - const int in_x = in_x_origin + dilation_width_factor * filter_x; - const int in_y = - in_y_origin + dilation_height_factor * filter_y; - // If the location is outside the bounds of the input image, - // use zero as a default value. - if ((in_x >= 0) && (in_x < input_width) && (in_y >= 0) && - (in_y < input_height)) { - float input_value = input_data[Offset( - input_shape, batch, in_y, in_x, in_channel)]; - float filter_value = - filter_data[Offset(filter_shape, out_channel, filter_y, - filter_x, in_channel)]; - total += (input_value * filter_value); - } + float input_value = input_data[Offset(input_shape, batch, in_y, + in_x, in_channel)]; + float filter_value = filter_data[Offset( + filter_shape, out_channel, filter_y, filter_x, in_channel)]; + total += (input_value * filter_value); } } } @@ -99,11 +102,11 @@ inline void Conv(const ConvParams& params, const RuntimeShape& input_shape, } inline void Conv(const ConvParams& params, const RuntimeShape& input_shape, - const uint8* input_data, const RuntimeShape& filter_shape, - const uint8* filter_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, - uint8* output_data, const RuntimeShape& im2col_shape, - uint8* im2col_data, void* cpu_backend_context) { + const uint8_t* input_data, const RuntimeShape& filter_shape, + const uint8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + uint8_t* output_data, const RuntimeShape& im2col_shape, + uint8_t* im2col_data, void* cpu_backend_context) { (void)cpu_backend_context; // only used in optimized code. (void)im2col_data; // only used in optimized code. (void)im2col_shape; // only used in optimized code. @@ -113,13 +116,13 @@ inline void Conv(const ConvParams& params, const RuntimeShape& input_shape, const int dilation_height_factor = params.dilation_height_factor; const int pad_width = params.padding_values.width; const int pad_height = params.padding_values.height; - const int32 input_offset = params.input_offset; - const int32 filter_offset = params.weights_offset; - const int32 output_offset = params.output_offset; - const int32 output_multiplier = params.output_multiplier; + const int32_t input_offset = params.input_offset; + const int32_t filter_offset = params.weights_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_multiplier = params.output_multiplier; const int output_shift = params.output_shift; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; TFLITE_DCHECK_LE(output_activation_min, output_activation_max); TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); @@ -139,29 +142,32 @@ inline void Conv(const ConvParams& params, const RuntimeShape& input_shape, const int output_width = output_shape.Dims(2); for (int batch = 0; batch < batches; ++batch) { for (int out_y = 0; out_y < output_height; ++out_y) { + const int in_y_origin = (out_y * stride_height) - pad_height; for (int out_x = 0; out_x < output_width; ++out_x) { + const int in_x_origin = (out_x * stride_width) - pad_width; for (int out_channel = 0; out_channel < output_depth; ++out_channel) { - const int in_x_origin = (out_x * stride_width) - pad_width; - const int in_y_origin = (out_y * stride_height) - pad_height; - int32 acc = 0; + int32_t acc = 0; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { + const int in_y = in_y_origin + dilation_height_factor * filter_y; for (int filter_x = 0; filter_x < filter_width; ++filter_x) { + const int in_x = in_x_origin + dilation_width_factor * filter_x; + + // Zero padding by omitting the areas outside the image. + const bool is_point_inside_image = + (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && + (in_y < input_height); + + if (!is_point_inside_image) { + continue; + } + for (int in_channel = 0; in_channel < input_depth; ++in_channel) { - const int in_x = in_x_origin + dilation_width_factor * filter_x; - const int in_y = - in_y_origin + dilation_height_factor * filter_y; - // If the location is outside the bounds of the input image, - // use zero as a default value. - if ((in_x >= 0) && (in_x < input_width) && (in_y >= 0) && - (in_y < input_height)) { - int32 input_val = input_data[Offset(input_shape, batch, in_y, + int32_t input_val = input_data[Offset(input_shape, batch, in_y, in_x, in_channel)]; - int32 filter_val = - filter_data[Offset(filter_shape, out_channel, filter_y, - filter_x, in_channel)]; - acc += - (filter_val + filter_offset) * (input_val + input_offset); - } + int32_t filter_val = filter_data[Offset( + filter_shape, out_channel, filter_y, filter_x, in_channel)]; + acc += + (filter_val + filter_offset) * (input_val + input_offset); } } } @@ -174,7 +180,7 @@ inline void Conv(const ConvParams& params, const RuntimeShape& input_shape, acc = std::max(acc, output_activation_min); acc = std::min(acc, output_activation_max); output_data[Offset(output_shape, batch, out_y, out_x, out_channel)] = - static_cast(acc); + static_cast(acc); } } } @@ -220,7 +226,7 @@ inline void HybridConvPerChannel( for (int out_channel = 0; out_channel < output_depth; ++out_channel) { const int in_x_origin = (out_x * stride_width) - pad_width; const int in_y_origin = (out_y * stride_height) - pad_height; - int32 acc = 0; + int32_t acc = 0; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { for (int filter_x = 0; filter_x < filter_width; ++filter_x) { for (int in_channel = 0; in_channel < input_depth; ++in_channel) { @@ -231,9 +237,9 @@ inline void HybridConvPerChannel( // use zero as a default value. if ((in_x >= 0) && (in_x < input_width) && (in_y >= 0) && (in_y < input_height)) { - int32 input_val = input_data[Offset(input_shape, batch, in_y, - in_x, in_channel)]; - int32 filter_val = + int32_t input_val = input_data[Offset( + input_shape, batch, in_y, in_x, in_channel)]; + int32_t filter_val = filter_data[Offset(filter_shape, out_channel, filter_y, filter_x, in_channel)]; acc += filter_val * (input_val - input_offset[batch]); @@ -258,5 +264,4 @@ inline void HybridConvPerChannel( } // namespace reference_ops } // namespace tflite - #endif // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_CONV_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h index 70e5dd40..20bf83df 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h @@ -62,21 +62,21 @@ namespace reference_ops { namespace depthwise_conv { template -inline int32 DepthwiseConvRound(int32 x, int32 quantized_multiplier, - int shift) { +inline int32_t DepthwiseConvRound(int32_t x, int32_t quantized_multiplier, + int shift) { TFLITE_DCHECK_NE(output_rounding, DepthwiseConvOutputRounding::kNone); return MultiplyByQuantizedMultiplier(x, quantized_multiplier, shift); } template <> -inline int32 DepthwiseConvRound( - int32 x, int32 quantized_multiplier, int shift) { +inline int32_t DepthwiseConvRound( + int32_t x, int32_t quantized_multiplier, int shift) { return MultiplyByQuantizedMultiplier(x, quantized_multiplier, shift); } template <> -inline int32 DepthwiseConvRound( - int32 x, int32 quantized_multiplier, int shift) { +inline int32_t DepthwiseConvRound( + int32_t x, int32_t quantized_multiplier, int shift) { using gemmlowp::SaturatingRoundingDoublingHighMul; const int left_shift = shift > 0 ? shift : 0; const int right_shift = shift > 0 ? 0 : -shift; @@ -89,13 +89,12 @@ inline int32 DepthwiseConvRound( template struct DepthwiseConvBasicKernel { - static inline void Run(const DepthwiseParams& params, - const RuntimeShape& input_shape, - const uint8* input_data, - const RuntimeShape& filter_shape, - const uint8* filter_data, - const RuntimeShape& bias_shape, const int32* bias_data, - const RuntimeShape& output_shape, uint8* output_data) { + static inline void Run( + const DepthwiseParams& params, const RuntimeShape& input_shape, + const uint8_t* input_data, const RuntimeShape& filter_shape, + const uint8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + uint8_t* output_data) { const int stride_width = params.stride_width; const int stride_height = params.stride_height; const int dilation_width_factor = params.dilation_width_factor; @@ -103,12 +102,12 @@ struct DepthwiseConvBasicKernel { const int pad_width = params.padding_values.width; const int pad_height = params.padding_values.height; const int depth_multiplier = params.depth_multiplier; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; - const int32 input_offset = params.input_offset; - const int32 filter_offset = params.weights_offset; - const int32 output_offset = params.output_offset; - const int32 output_multiplier = params.output_multiplier; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; + const int32_t input_offset = params.input_offset; + const int32_t filter_offset = params.weights_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_multiplier = params.output_multiplier; const int output_shift = params.output_shift; TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4); @@ -135,7 +134,7 @@ struct DepthwiseConvBasicKernel { const int oc = m + ic * depth_multiplier; const int in_x_origin = (out_x * stride_width) - pad_width; const int in_y_origin = (out_y * stride_height) - pad_height; - int32 acc = 0; + int32_t acc = 0; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { for (int filter_x = 0; filter_x < filter_width; ++filter_x) { const int in_x = @@ -146,9 +145,9 @@ struct DepthwiseConvBasicKernel { // use zero as a default value. if ((in_x >= 0) && (in_x < input_width) && (in_y >= 0) && (in_y < input_height)) { - int32 input_val = + int32_t input_val = input_data[Offset(input_shape, b, in_y, in_x, ic)]; - int32 filter_val = filter_data[Offset( + int32_t filter_val = filter_data[Offset( filter_shape, 0, filter_y, filter_x, oc)]; acc += (filter_val + filter_offset) * (input_val + input_offset); @@ -164,7 +163,7 @@ struct DepthwiseConvBasicKernel { acc = std::max(acc, output_activation_min); acc = std::min(acc, output_activation_max); output_data[Offset(output_shape, b, out_y, out_x, oc)] = - static_cast(acc); + static_cast(acc); } } } @@ -176,10 +175,10 @@ struct DepthwiseConvBasicKernel { // MultiplyByQuantizedMultiplier or DepthwiseConvRound function. static inline void RunPerChannel( const DepthwiseParams& params, const RuntimeShape& input_shape, - const int8* input_data, const RuntimeShape& filter_shape, - const int8* filter_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, - int8* output_data) { + const int8_t* input_data, const RuntimeShape& filter_shape, + const int8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + int8_t* output_data) { // Get parameters. // TODO(b/141565753): Re-introduce ScopedProfilingLabel on Micro. const int stride_width = params.stride_width; @@ -189,12 +188,12 @@ struct DepthwiseConvBasicKernel { const int pad_width = params.padding_values.width; const int pad_height = params.padding_values.height; const int depth_multiplier = params.depth_multiplier; - const int32 input_offset = params.input_offset; - const int32 output_offset = params.output_offset; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; - const int32* output_multiplier = params.output_multiplier_per_channel; - const int32* output_shift = params.output_shift_per_channel; + const int32_t input_offset = params.input_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; + const int32_t* output_multiplier = params.output_multiplier_per_channel; + const int32_t* output_shift = params.output_shift_per_channel; // Check dimensions of the tensors. TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); @@ -222,7 +221,7 @@ struct DepthwiseConvBasicKernel { const int output_channel = m + in_channel * depth_multiplier; const int in_x_origin = (out_x * stride_width) - pad_width; const int in_y_origin = (out_y * stride_height) - pad_height; - int32 acc = 0; + int32_t acc = 0; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { for (int filter_x = 0; filter_x < filter_width; ++filter_x) { const int in_x = @@ -234,17 +233,18 @@ struct DepthwiseConvBasicKernel { (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && (in_y < input_height); if (is_point_inside_image) { - int32 input_val = input_data[Offset( + int32_t input_val = input_data[Offset( input_shape, batch, in_y, in_x, in_channel)]; - int32 filter_val = filter_data[Offset( + int32_t filter_val = filter_data[Offset( filter_shape, 0, filter_y, filter_x, output_channel)]; // Accumulate with 32 bits accumulator. // In the nudging process during model quantization, we // force real value of 0.0 be represented by a quantized - // value. This guarantees that the input_offset is a int8, - // even though it is represented using int32. int32 += int8 - // * (int8 - int8) so the highest value we can get from each - // accumulation is [-127, 127] * ([-128, 127] - + // value. This guarantees that the input_offset is a int8_t, + // even though it is represented using int32_t. int32_t += + // int8_t + // * (int8_t - int8_t) so the highest value we can get from + // each accumulation is [-127, 127] * ([-128, 127] - // [-128, 127]), which is [-32512, 32512]. log2(32512) // = 14.98, which means we can accumulate at least 2^16 // multiplications without overflow. The accumulator is @@ -279,10 +279,10 @@ struct DepthwiseConvBasicKernel { inline void DepthwiseConv( const DepthwiseParams& params, const RuntimeShape& input_shape, - const uint8* input_data, const RuntimeShape& filter_shape, - const uint8* filter_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, - uint8* output_data) { + const uint8_t* input_data, const RuntimeShape& filter_shape, + const uint8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + uint8_t* output_data) { return depthwise_conv::DepthwiseConvBasicKernel< DepthwiseConvOutputRounding::kAwayFromZero>::Run(params, input_shape, input_data, filter_shape, diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/dequantize.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/dequantize.h index 286c9310..b90951f9 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/dequantize.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/dequantize.h @@ -32,12 +32,12 @@ inline void Dequantize(const tflite::DequantizationParams& op_params, const RuntimeShape& input_shape, const InputT* input_data, const RuntimeShape& output_shape, OutputT* output_data) { - int32 zero_point = op_params.zero_point; + int32_t zero_point = op_params.zero_point; const double scale = op_params.scale; const int flat_size = MatchingFlatSize(input_shape, output_shape); for (int i = 0; i < flat_size; i++) { - const int32 val = input_data[i]; + const int32_t val = input_data[i]; const OutputT result = static_cast(scale * (val - zero_point)); output_data[i] = result; } @@ -52,11 +52,11 @@ inline void PerChannelDequantize( // Ensure flat size is same. MatchingFlatSize(input_shape, output_shape); - const int32* zero_point = op_params.zero_point; + const int32_t* zero_point = op_params.zero_point; const float* scale = op_params.scale; - const int32 quantized_dimension = op_params.quantized_dimension; - const int32 num_dims = input_shape.DimensionsCount(); - const int32* dims_data = input_shape.DimsData(); + const int32_t quantized_dimension = op_params.quantized_dimension; + const int32_t num_dims = input_shape.DimensionsCount(); + const int32_t* dims_data = input_shape.DimsData(); std::vector current_dim(num_dims, 0); do { @@ -64,7 +64,7 @@ inline void PerChannelDequantize( ReducedOutputOffset(num_dims, reinterpret_cast(dims_data), current_dim.data(), 0, nullptr); const int channel = current_dim[quantized_dimension]; - const int32 val = input_data[offset]; + const int32_t val = input_data[offset]; const float result = static_cast(scale[channel] * (val - zero_point[channel])); output_data[offset] = result; diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/fully_connected.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/fully_connected.h index 204a0fa0..39a9cd02 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/fully_connected.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/fully_connected.h @@ -61,17 +61,17 @@ inline void FullyConnected( inline void FullyConnected( const FullyConnectedParams& params, const RuntimeShape& input_shape, - const uint8* input_data, const RuntimeShape& filter_shape, - const uint8* filter_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, - uint8* output_data) { - const int32 input_offset = params.input_offset; - const int32 filter_offset = params.weights_offset; - const int32 output_offset = params.output_offset; - const int32 output_multiplier = params.output_multiplier; + const uint8_t* input_data, const RuntimeShape& filter_shape, + const uint8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + uint8_t* output_data) { + const int32_t input_offset = params.input_offset; + const int32_t filter_offset = params.weights_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_multiplier = params.output_multiplier; const int output_shift = params.output_shift; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; TFLITE_DCHECK_GE(filter_shape.DimensionsCount(), 2); TFLITE_DCHECK_GE(output_shape.DimensionsCount(), 1); @@ -89,10 +89,10 @@ inline void FullyConnected( const int accum_depth = filter_shape.Dims(filter_dim_count - 1); for (int b = 0; b < batches; ++b) { for (int out_c = 0; out_c < output_depth; ++out_c) { - int32 acc = 0; + int32_t acc = 0; for (int d = 0; d < accum_depth; ++d) { - int32 input_val = input_data[b * accum_depth + d]; - int32 filter_val = filter_data[out_c * accum_depth + d]; + int32_t input_val = input_data[b * accum_depth + d]; + int32_t filter_val = filter_data[out_c * accum_depth + d]; acc += (filter_val + filter_offset) * (input_val + input_offset); } if (bias_data) { @@ -102,24 +102,24 @@ inline void FullyConnected( acc += output_offset; acc = std::max(acc, output_activation_min); acc = std::min(acc, output_activation_max); - output_data[out_c + output_depth * b] = static_cast(acc); + output_data[out_c + output_depth * b] = static_cast(acc); } } } inline void FullyConnected( const FullyConnectedParams& params, const RuntimeShape& input_shape, - const uint8* input_data, const RuntimeShape& filter_shape, - const uint8* filter_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, - int16* output_data) { - const int32 input_offset = params.input_offset; - const int32 filter_offset = params.weights_offset; - const int32 output_offset = params.output_offset; - const int32 output_multiplier = params.output_multiplier; + const uint8_t* input_data, const RuntimeShape& filter_shape, + const uint8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + int16_t* output_data) { + const int32_t input_offset = params.input_offset; + const int32_t filter_offset = params.weights_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_multiplier = params.output_multiplier; const int output_shift = params.output_shift; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; TFLITE_DCHECK_LE(output_activation_min, output_activation_max); TFLITE_DCHECK_EQ(output_offset, 0); @@ -138,20 +138,21 @@ inline void FullyConnected( for (int out_c = 0; out_c < output_depth; ++out_c) { // Internal accumulation. // Initialize accumulator with the bias-value. - int32 accum = bias_data[out_c]; + int32_t accum = bias_data[out_c]; // Accumulation loop. for (int d = 0; d < accum_depth; ++d) { - int16 input_val = input_data[b * accum_depth + d] + input_offset; - int16 filter_val = filter_data[out_c * accum_depth + d] + filter_offset; + int16_t input_val = input_data[b * accum_depth + d] + input_offset; + int16_t filter_val = + filter_data[out_c * accum_depth + d] + filter_offset; accum += filter_val * input_val; } - // Down-scale the final int32 accumulator to the scale used by our + // Down-scale the final int32_t accumulator to the scale used by our // (16-bit, typically 3 integer bits) fixed-point format. The quantized // multiplier and shift here have been pre-computed offline // (e.g. by toco). accum = MultiplyByQuantizedMultiplier(accum, output_multiplier, output_shift); - // Saturate, cast to int16, and store to output array. + // Saturate, cast to int16_t, and store to output array. accum = std::max(accum, output_activation_min - output_offset); accum = std::min(accum, output_activation_max - output_offset); accum += output_offset; @@ -162,14 +163,14 @@ inline void FullyConnected( inline void ShuffledFullyConnected( const FullyConnectedParams& params, const RuntimeShape& input_shape, - const uint8* input_data, const RuntimeShape& weights_shape, - const uint8* shuffled_weights_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, - int16* output_data, uint8* shuffled_input_workspace_data) { - const int32 output_multiplier = params.output_multiplier; + const uint8_t* input_data, const RuntimeShape& weights_shape, + const uint8_t* shuffled_weights_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + int16_t* output_data, uint8_t* shuffled_input_workspace_data) { + const int32_t output_multiplier = params.output_multiplier; const int output_shift = params.output_shift; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; TFLITE_DCHECK_LE(output_activation_min, output_activation_max); TFLITE_DCHECK_GE(input_shape.DimensionsCount(), 1); @@ -190,7 +191,7 @@ inline void ShuffledFullyConnected( TFLITE_DCHECK((output_depth % 4) == 0); // Shuffling and xoring of input activations into the workspace buffer - uint8* shuffled_input_workspace_ptr = shuffled_input_workspace_data; + uint8_t* shuffled_input_workspace_ptr = shuffled_input_workspace_data; if (batches == 1) { for (int i = 0; i < accum_depth; i++) { shuffled_input_workspace_data[i] = input_data[i] ^ 0x80; @@ -198,13 +199,13 @@ inline void ShuffledFullyConnected( } else if (batches == 4) { for (int c = 0; c < accum_depth; c += 16) { for (int b = 0; b < 4; b++) { - const uint8* src_data_ptr = input_data + b * accum_depth + c; + const uint8_t* src_data_ptr = input_data + b * accum_depth + c; for (int j = 0; j < 16; j++) { - uint8 src_val = *src_data_ptr++; + uint8_t src_val = *src_data_ptr++; // Flip the sign bit, so that the kernel will only need to - // reinterpret these uint8 values as int8, getting for free the + // reinterpret these uint8_t values as int8_t, getting for free the // subtraction of the zero_point value 128. - uint8 dst_val = src_val ^ 0x80; + uint8_t dst_val = src_val ^ 0x80; *shuffled_input_workspace_ptr++ = dst_val; } } @@ -216,62 +217,62 @@ inline void ShuffledFullyConnected( // Actual computation if (batches == 1) { - int16* output_ptr = output_data; + int16_t* output_ptr = output_data; // Shuffled weights have had their sign bit (0x80) pre-flipped (xor'd) - // so that just reinterpreting them as int8 values is equivalent to + // so that just reinterpreting them as int8_t values is equivalent to // subtracting 128 from them, thus implementing for free the subtraction of // the zero_point value 128. - const int8* shuffled_weights_ptr = - reinterpret_cast(shuffled_weights_data); + const int8_t* shuffled_weights_ptr = + reinterpret_cast(shuffled_weights_data); // Likewise, we preshuffled and pre-xored the input data above. - const int8* shuffled_input_data = - reinterpret_cast(shuffled_input_workspace_data); + const int8_t* shuffled_input_data = + reinterpret_cast(shuffled_input_workspace_data); for (int c = 0; c < output_depth; c += 4) { // Internal accumulation. // Initialize accumulator with the bias-value. - int32 accum[4] = {0}; + int32_t accum[4] = {0}; // Accumulation loop. for (int d = 0; d < accum_depth; d += 16) { for (int i = 0; i < 4; i++) { for (int j = 0; j < 16; j++) { - int8 input_val = shuffled_input_data[d + j]; - int8 weights_val = *shuffled_weights_ptr++; + int8_t input_val = shuffled_input_data[d + j]; + int8_t weights_val = *shuffled_weights_ptr++; accum[i] += weights_val * input_val; } } } for (int i = 0; i < 4; i++) { // Add bias value - int32 acc = accum[i] + bias_data[c + i]; - // Down-scale the final int32 accumulator to the scale used by our + int32_t acc = accum[i] + bias_data[c + i]; + // Down-scale the final int32_t accumulator to the scale used by our // (16-bit, typically 3 integer bits) fixed-point format. The quantized // multiplier and shift here have been pre-computed offline // (e.g. by toco). acc = MultiplyByQuantizedMultiplier(acc, output_multiplier, output_shift); - // Saturate, cast to int16, and store to output array. + // Saturate, cast to int16_t, and store to output array. acc = std::max(acc, output_activation_min); acc = std::min(acc, output_activation_max); output_ptr[c + i] = acc; } } } else if (batches == 4) { - int16* output_ptr = output_data; + int16_t* output_ptr = output_data; // Shuffled weights have had their sign bit (0x80) pre-flipped (xor'd) - // so that just reinterpreting them as int8 values is equivalent to + // so that just reinterpreting them as int8_t values is equivalent to // subtracting 128 from them, thus implementing for free the subtraction of // the zero_point value 128. - const int8* shuffled_weights_ptr = - reinterpret_cast(shuffled_weights_data); + const int8_t* shuffled_weights_ptr = + reinterpret_cast(shuffled_weights_data); // Likewise, we preshuffled and pre-xored the input data above. - const int8* shuffled_input_data = - reinterpret_cast(shuffled_input_workspace_data); + const int8_t* shuffled_input_data = + reinterpret_cast(shuffled_input_workspace_data); for (int c = 0; c < output_depth; c += 4) { - const int8* shuffled_input_ptr = shuffled_input_data; + const int8_t* shuffled_input_ptr = shuffled_input_data; // Accumulation loop. // Internal accumulation. // Initialize accumulator with the bias-value. - int32 accum[4][4]; + int32_t accum[4][4]; for (int i = 0; i < 4; i++) { for (int b = 0; b < 4; b++) { accum[i][b] = 0; @@ -281,8 +282,8 @@ inline void ShuffledFullyConnected( for (int i = 0; i < 4; i++) { for (int b = 0; b < 4; b++) { for (int j = 0; j < 16; j++) { - int8 input_val = shuffled_input_ptr[16 * b + j]; - int8 weights_val = shuffled_weights_ptr[16 * i + j]; + int8_t input_val = shuffled_input_ptr[16 * b + j]; + int8_t weights_val = shuffled_weights_ptr[16 * i + j]; accum[i][b] += weights_val * input_val; } } @@ -293,14 +294,14 @@ inline void ShuffledFullyConnected( for (int i = 0; i < 4; i++) { for (int b = 0; b < 4; b++) { // Add bias value - int32 acc = accum[i][b] + bias_data[c + i]; - // Down-scale the final int32 accumulator to the scale used by our + int32_t acc = accum[i][b] + bias_data[c + i]; + // Down-scale the final int32_t accumulator to the scale used by our // (16-bit, typically 3 integer bits) fixed-point format. The // quantized multiplier and shift here have been pre-computed offline // (e.g. by toco). acc = MultiplyByQuantizedMultiplier(acc, output_multiplier, output_shift); - // Saturate, cast to int16, and store to output array. + // Saturate, cast to int16_t, and store to output array. acc = std::max(acc, output_activation_min); acc = std::min(acc, output_activation_max); output_ptr[b * output_depth + c + i] = acc; diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/hard_swish.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/hard_swish.h new file mode 100644 index 00000000..cda1b5cf --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/hard_swish.h @@ -0,0 +1,166 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_ACTIVATIONS_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_ACTIVATIONS_H_ + +#include "ruy/profiler/instrumentation.h" // from @ruy +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/types.h" + +namespace tflite { +namespace reference_ops { + +inline int16_t SaturatingLeftShift(int16_t value, int amount) { + int32_t result = static_cast(value) * (1 << amount); + result = std::min(result, std::numeric_limits::max()); + result = std::max(result, std::numeric_limits::min()); + return result; +} + +// Similar to ARM instruction SQDMULH. +// Similar to gemmlowp::SaturatingRoundingDoublingHighMul except +// rounding to zero instead of to nearest (SQRDMULH). +inline std::int16_t SaturatingDoublingHighMul(std::int16_t a, std::int16_t b) { + bool overflow = a == b && a == std::numeric_limits::min(); + std::int32_t a_32(a); + std::int32_t b_32(b); + std::int32_t ab_32 = a_32 * b_32; + std::int16_t ab_x2_high16 = static_cast((ab_32) / (1 << 15)); + return overflow ? std::numeric_limits::max() : ab_x2_high16; +} + +template +inline void HardSwish(const RuntimeShape& input_shape, const T* input_data, + const RuntimeShape& output_shape, T* output_data) { + ruy::profiler::ScopeLabel label("ReferenceHardSwish/Float"); + auto matching_size = MatchingFlatSize(input_shape, output_shape); + const T* in_end = input_data + matching_size; + for (; input_data < in_end; input_data++, output_data++) { + const float in = *input_data; + *output_data = + in * std::min(static_cast(6), std::max(static_cast(0), in + 3)) / + 6; + } +} + +template +inline void HardSwish(const HardSwishParams& params, + const RuntimeShape& input_shape, const T* input_data, + const RuntimeShape& output_shape, T* output_data) { + ruy::profiler::ScopeLabel label("ReferenceHardSwish/Quantized"); + + const int flat_size = MatchingFlatSize(input_shape, output_shape); + + for (int i = 0; i < flat_size; i++) { + const int16_t input_value = input_data[i] - params.input_zero_point; + // Left-shift as much as we can without overflow/saturation to put + // significant bits in the high bits of our 16-bit fixedpoint values, so + // that fixed-point approximate computations below are as accurate as + // possible. + const int16_t input_value_on_hires_input_scale = input_value * (1 << 7); + // Compute the input value on essentially the output scale, just not + // right-shifted yet. This is the value that we'll use in the (x >= +3) + // case, and that in the general case we'll multiply against the "relu-ish" + // fixed-point multiplier in [0, 1]. + const int16_t input_value_on_preshift_output_scale = + gemmlowp::SaturatingRoundingDoublingHighMul( + input_value_on_hires_input_scale, + params.output_multiplier_fixedpoint_int16); + // Now compute the "relu-ish multiplier". In the (-3 <= x <= +3) case, that + // is just an affine rescaling of x from [-3, 3] to [0, 1]. In the general + // case, it is just that plus saturation at the boundaries of [-3, 3]. + // First, we rescale from [-3, 3] to [-1, 1], saturating. + // That is done by rescaling the input value with a fixed-point multiplier + // (reluish_multiplier_fixedpoint) and bit-shift such that we represent + // that input value on the scale where the real value 3.0f is represented + // by the quantized value 32768. (+32768 is actually not representable as + // int16_t, so this saturates at +32767, and that is seen empirically to be + // a negligible contribution to numerical error/bias). + // + // This code is careful to correctly implement any magnitude of multiplier, + // involving either a right shift or a left shift, with correct saturation + // behavior in the left-shift case. This forces this code to be more + // complicated, but is necessary for real applications: a partially + // trained quantized MobileNet v3-small model that motivated this code + // exhibits some large [min, max] range boundaries, of the order of + // magnitude of 10 or 100 depending on layers. + // + // The next few lines are basically just an ordinary + // MultiplyByQuantizedMultiplier, except that we are more careful here + // about the fine details of saturation when left-shifting, because here + // overflow in left-shift is a common case, not an anomaly as + // MultiplyByQuantizedMultiplier assumes. + int16_t reluish_value = input_value_on_hires_input_scale; + // Shift left, saturating, as much as we can while ensuring that this + // saturation will not contribute to the result. That is, left shift amount + // reduced by 1. + if (params.reluish_multiplier_exponent > 0) { + reluish_value = SaturatingLeftShift( + reluish_value, params.reluish_multiplier_exponent - 1); + } + // Apply the fixed-point multiplier, dividing the value by a divisor + // ranging in [1, 2]. + reluish_value = gemmlowp::SaturatingRoundingDoublingHighMul( + reluish_value, params.reluish_multiplier_fixedpoint_int16); + // Apply the last bit of left-shift. Thus, in the left-shifting case, if + // any saturation affects the result, it is happening here --- any + // saturation having occurred above is overwritten here, not affecting the + // result. + if (params.reluish_multiplier_exponent > 0) { + reluish_value = SaturatingLeftShift(reluish_value, 1); + } + // Shift right, in the right-shifting case. + if (params.reluish_multiplier_exponent < 0) { + reluish_value = gemmlowp::RoundingDivideByPOT( + reluish_value, -params.reluish_multiplier_exponent); + } + // At this point we have rescaled the value into a 16bit fixedpoint + // reluish_value in [-1, 1]. + // We now convert that to a 16bit fixedpoint value in [0, 1]. + reluish_value = (reluish_value + (1 << 15)) >> 1; + // Use of SaturatingDoublingHighMul here is important to cancel the biases + // from the above SaturatingRoundingDoublingHighMul. + // + // On a partially trained MobileNet-v3-small, + // + // | bias on | ImageNet + // | quantized | Top-1 + // Operation used here | values | accuracy (50k) + // --------------------------------------+------------+----------- + // SaturatingDoublingHighMul | -0.0024 | 58.920 + // SaturatingRoundingDoublingHighMul | -0.0067 | 58.064 + // + // In activations_test, this is covered by this testcase: + // QuantizedActivationsOpTest.HardSwishBias + // + const int16_t preshift_output_value = SaturatingDoublingHighMul( + reluish_value, input_value_on_preshift_output_scale); + // We were so far operating on the pre-shift output scale. Now we finally + // apply that output shift, arriving at the final output scale. + int16_t output_value = gemmlowp::RoundingDivideByPOT( + preshift_output_value, -params.output_multiplier_exponent); + output_value += params.output_zero_point; + output_value = + std::min(output_value, std::numeric_limits::max()); + output_value = + std::max(output_value, std::numeric_limits::min()); + output_data[i] = output_value; + } +} + +} // namespace reference_ops +} // namespace tflite + +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_CONV_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/add.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/add.h index 69b42e08..2af6f373 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/add.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/add.h @@ -23,34 +23,41 @@ limitations under the License. namespace tflite { namespace reference_integer_ops { +inline void CheckArithmeticParams(const ArithmeticParams& params) { + TFLITE_DCHECK_LE(params.quantized_activation_min, + params.quantized_activation_max); + // Input offset is negative input zero point. Activation tensors are + // asymmetric quantized so they span the full int8 range. + TFLITE_DCHECK_GE(-params.input1_offset, std::numeric_limits::min()); + TFLITE_DCHECK_GE(-params.input2_offset, std::numeric_limits::min()); + TFLITE_DCHECK_LE(-params.input1_offset, std::numeric_limits::max()); + TFLITE_DCHECK_LE(-params.input2_offset, std::numeric_limits::max()); +} + // Element-wise add that can often be used for inner loop of broadcast add as // well as the non-broadcast add. inline void AddElementwise(int size, const ArithmeticParams& params, const int8_t* input1_data, const int8_t* input2_data, int8_t* output_data) { - const int32_t int8_max_value = std::numeric_limits::max(); - TFLITE_DCHECK_GE(params.input1_offset, -1 * int8_max_value); - TFLITE_DCHECK_GE(params.input2_offset, -1 * int8_max_value); - TFLITE_DCHECK_LE(params.input1_offset, int8_max_value); - TFLITE_DCHECK_LE(params.input2_offset, int8_max_value); + CheckArithmeticParams(params); for (int i = 0; i < size; ++i) { - const int32 input1_val = params.input1_offset + input1_data[i]; - const int32 input2_val = params.input2_offset + input2_data[i]; - const int32 shifted_input1_val = input1_val * (1 << params.left_shift); - const int32 shifted_input2_val = input2_val * (1 << params.left_shift); - const int32 scaled_input1_val = + const int32_t input1_val = params.input1_offset + input1_data[i]; + const int32_t input2_val = params.input2_offset + input2_data[i]; + const int32_t shifted_input1_val = input1_val * (1 << params.left_shift); + const int32_t shifted_input2_val = input2_val * (1 << params.left_shift); + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, params.input1_multiplier, params.input1_shift); - const int32 scaled_input2_val = + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, params.input2_multiplier, params.input2_shift); - const int32 raw_sum = scaled_input1_val + scaled_input2_val; - const int32 raw_output = + const int32_t raw_sum = scaled_input1_val + scaled_input2_val; + const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp( raw_sum, params.output_multiplier, params.output_shift) + params.output_offset; - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); output_data[i] = static_cast(clamped_output); @@ -61,16 +68,11 @@ inline void Add(const ArithmeticParams& params, const RuntimeShape& input1_shape, const int8_t* input1_data, const RuntimeShape& input2_shape, const int8_t* input2_data, const RuntimeShape& output_shape, int8_t* output_data) { - TFLITE_DCHECK_LE(params.quantized_activation_min, - params.quantized_activation_max); + CheckArithmeticParams(params); + const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape); - const int32_t int8_max_value = std::numeric_limits::max(); - TFLITE_DCHECK_GE(params.input1_offset, -1 * int8_max_value); - TFLITE_DCHECK_GE(params.input2_offset, -1 * int8_max_value); - TFLITE_DCHECK_LE(params.input1_offset, int8_max_value); - TFLITE_DCHECK_LE(params.input2_offset, int8_max_value); AddElementwise(flat_size, params, input1_data, input2_data, output_data); } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h index 9131c7db..3e9cd0ca 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h @@ -22,27 +22,27 @@ namespace reference_integer_ops { // Fixed-point per-channel-quantization convolution reference kernel. inline void ConvPerChannel( - const ConvParams& params, const int32* output_multiplier, - const int32* output_shift, const RuntimeShape& input_shape, - const int8* input_data, const RuntimeShape& filter_shape, - const int8* filter_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, - int8* output_data) { + const ConvParams& params, const int32_t* output_multiplier, + const int32_t* output_shift, const RuntimeShape& input_shape, + const int8_t* input_data, const RuntimeShape& filter_shape, + const int8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + int8_t* output_data) { // Get parameters. - const int32 input_offset = params.input_offset; // r = s(q - Z) + const int32_t input_offset = params.input_offset; // r = s(q - Z) const int stride_width = params.stride_width; const int stride_height = params.stride_height; const int dilation_width_factor = params.dilation_width_factor; const int dilation_height_factor = params.dilation_height_factor; const int pad_width = params.padding_values.width; const int pad_height = params.padding_values.height; - const int32 output_offset = params.output_offset; + const int32_t output_offset = params.output_offset; // Set min and max value of the output. - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; - // Sanity check. + // Consistency check. TFLITE_DCHECK_LE(output_activation_min, output_activation_max); TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4); @@ -63,45 +63,47 @@ inline void ConvPerChannel( const int output_width = output_shape.Dims(2); for (int batch = 0; batch < batches; ++batch) { for (int out_y = 0; out_y < output_height; ++out_y) { + const int in_y_origin = (out_y * stride_height) - pad_height; for (int out_x = 0; out_x < output_width; ++out_x) { + const int in_x_origin = (out_x * stride_width) - pad_width; for (int out_channel = 0; out_channel < output_depth; ++out_channel) { - const int in_x_origin = (out_x * stride_width) - pad_width; - const int in_y_origin = (out_y * stride_height) - pad_height; - int32 acc = 0; + int32_t acc = 0; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { + const int in_y = in_y_origin + dilation_height_factor * filter_y; for (int filter_x = 0; filter_x < filter_width; ++filter_x) { + const int in_x = in_x_origin + dilation_width_factor * filter_x; + + // Zero padding by omitting the areas outside the image. + const bool is_point_inside_image = + (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && + (in_y < input_height); + + if (!is_point_inside_image) { + continue; + } + for (int in_channel = 0; in_channel < input_depth; ++in_channel) { - const int in_x = in_x_origin + dilation_width_factor * filter_x; - const int in_y = - in_y_origin + dilation_height_factor * filter_y; - // Zero padding by omitting the areas outside the image. - const bool is_point_inside_image = - (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && - (in_y < input_height); - if (is_point_inside_image) { - int32 input_val = input_data[Offset(input_shape, batch, in_y, + int32_t input_val = input_data[Offset(input_shape, batch, in_y, in_x, in_channel)]; - int32 filter_val = - filter_data[Offset(filter_shape, out_channel, filter_y, - filter_x, in_channel)]; - // Accumulate with 32 bits accumulator. - // In the nudging process during model quantization, we force - // real value of 0.0 be represented by a quantized value. This - // guarantees that the input_offset is a int8, even though it - // is represented using int32. - // int32 += int8 * (int8 - int8) so the highest value we can - // get from each accumulation is [-127, 127] * ([-128, 127] - - // [-128, 127]), which is [-32512, 32512]. log2(32512) - // = 14.98, which means we can accumulate at least 2^16 - // multiplications without overflow. The accumulator is - // applied to a filter so the accumulation logic will hold as - // long as the filter size (filter_y * filter_x * in_channel) - // does not exceed 2^16, which is the case in all the models - // we have seen so far. - // TODO(jianlijianli): Add a check to make sure the - // accumulator depth is smaller than 2^16. - acc += filter_val * (input_val + input_offset); - } + int32_t filter_val = filter_data[Offset( + filter_shape, out_channel, filter_y, filter_x, in_channel)]; + // Accumulate with 32 bits accumulator. + // In the nudging process during model quantization, we force + // real value of 0.0 be represented by a quantized value. This + // guarantees that the input_offset is a int8_t, even though + // it is represented using int32_t. int32_t += int8_t * + // (int8_t - int8_t) so the highest value we can get from each + // accumulation is [-127, 127] * ([-128, 127] - + // [-128, 127]), which is [-32512, 32512]. log2(32512) + // = 14.98, which means we can accumulate at least 2^16 + // multiplications without overflow. The accumulator is + // applied to a filter so the accumulation logic will hold as + // long as the filter size (filter_y * filter_x * in_channel) + // does not exceed 2^16, which is the case in all the models + // we have seen so far. + // TODO(jianlijianli): Add a check to make sure the + // accumulator depth is smaller than 2^16. + acc += filter_val * (input_val + input_offset); } } } @@ -125,12 +127,12 @@ inline void ConvPerChannel( // Fixed-point per-channel-quantization convolution reference kernel. // 16-bit data and 8-bit filter inline void ConvPerChannel( - const ConvParams& params, const int32* output_multiplier, - const int32* output_shift, const RuntimeShape& input_shape, - const int16* input_data, const RuntimeShape& filter_shape, - const int8* filter_data, const RuntimeShape& bias_shape, + const ConvParams& params, const int32_t* output_multiplier, + const int32_t* output_shift, const RuntimeShape& input_shape, + const int16_t* input_data, const RuntimeShape& filter_shape, + const int8_t* filter_data, const RuntimeShape& bias_shape, const std::int64_t* bias_data, const RuntimeShape& output_shape, - int16* output_data) { + int16_t* output_data) { // Get parameters. const int stride_width = params.stride_width; const int stride_height = params.stride_height; @@ -140,10 +142,10 @@ inline void ConvPerChannel( const int pad_height = params.padding_values.height; // Set min and max value of the output. - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; - // Sanity check. + // Consistency check. TFLITE_DCHECK_LE(output_activation_min, output_activation_max); TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4); @@ -164,35 +166,37 @@ inline void ConvPerChannel( const int output_width = output_shape.Dims(2); for (int batch = 0; batch < batches; ++batch) { for (int out_y = 0; out_y < output_height; ++out_y) { + const int in_y_origin = (out_y * stride_height) - pad_height; for (int out_x = 0; out_x < output_width; ++out_x) { + const int in_x_origin = (out_x * stride_width) - pad_width; for (int out_channel = 0; out_channel < output_depth; ++out_channel) { - const int in_x_origin = (out_x * stride_width) - pad_width; - const int in_y_origin = (out_y * stride_height) - pad_height; std::int64_t acc = 0; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { + const int in_y = in_y_origin + dilation_height_factor * filter_y; for (int filter_x = 0; filter_x < filter_width; ++filter_x) { + const int in_x = in_x_origin + dilation_width_factor * filter_x; + + // Zero padding by omitting the areas outside the image. + const bool is_point_inside_image = + (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && + (in_y < input_height); + + if (!is_point_inside_image) { + continue; + } + for (int in_channel = 0; in_channel < input_depth; ++in_channel) { - const int in_x = in_x_origin + dilation_width_factor * filter_x; - const int in_y = - in_y_origin + dilation_height_factor * filter_y; - // Zero padding by omitting the areas outside the image. - const bool is_point_inside_image = - (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && - (in_y < input_height); - if (is_point_inside_image) { - int32 input_val = input_data[Offset(input_shape, batch, in_y, + int32_t input_val = input_data[Offset(input_shape, batch, in_y, in_x, in_channel)]; - int32 filter_val = - filter_data[Offset(filter_shape, out_channel, filter_y, - filter_x, in_channel)]; - // Accumulate with 64 bits accumulator. - // int64 += int8 * int16 so the highest value we can - // get from each accumulation is [-127, 127] * ([-32768, - // 32767] - - // [-32768, 32767]), which is [-8322945, 8322945]. - // log2(8322945) = 22.99. - acc += filter_val * input_val; - } + int32_t filter_val = filter_data[Offset( + filter_shape, out_channel, filter_y, filter_x, in_channel)]; + // Accumulate with 64 bits accumulator. + // int64_t += int8_t * int16_t so the highest value we can + // get from each accumulation is [-127, 127] * ([-32768, + // 32767] - + // [-32768, 32767]), which is [-8322945, 8322945]. + // log2(8322945) = 22.99. + acc += filter_val * input_val; } } } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h index a4e00981..6f54e47f 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h @@ -20,12 +20,12 @@ limitations under the License. namespace tflite { namespace reference_integer_ops { inline void DepthwiseConvPerChannel( - const DepthwiseParams& params, const int32* output_multiplier, - const int32* output_shift, const RuntimeShape& input_shape, - const int8* input_data, const RuntimeShape& filter_shape, - const int8* filter_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, - int8* output_data) { + const DepthwiseParams& params, const int32_t* output_multiplier, + const int32_t* output_shift, const RuntimeShape& input_shape, + const int8_t* input_data, const RuntimeShape& filter_shape, + const int8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + int8_t* output_data) { // Get parameters. // TODO(b/141565753): Re-introduce ScopedProfilingLabel on Micro. const int stride_width = params.stride_width; @@ -35,10 +35,10 @@ inline void DepthwiseConvPerChannel( const int pad_width = params.padding_values.width; const int pad_height = params.padding_values.height; const int depth_multiplier = params.depth_multiplier; - const int32 input_offset = params.input_offset; - const int32 output_offset = params.output_offset; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t input_offset = params.input_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; // Check dimensions of the tensors. TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); @@ -66,7 +66,7 @@ inline void DepthwiseConvPerChannel( const int output_channel = m + in_channel * depth_multiplier; const int in_x_origin = (out_x * stride_width) - pad_width; const int in_y_origin = (out_y * stride_height) - pad_height; - int32 acc = 0; + int32_t acc = 0; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { for (int filter_x = 0; filter_x < filter_width; ++filter_x) { const int in_x = in_x_origin + dilation_width_factor * filter_x; @@ -77,17 +77,17 @@ inline void DepthwiseConvPerChannel( (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && (in_y < input_height); if (is_point_inside_image) { - int32 input_val = input_data[Offset(input_shape, batch, in_y, - in_x, in_channel)]; - int32 filter_val = filter_data[Offset( + int32_t input_val = input_data[Offset( + input_shape, batch, in_y, in_x, in_channel)]; + int32_t filter_val = filter_data[Offset( filter_shape, 0, filter_y, filter_x, output_channel)]; // Accumulate with 32 bits accumulator. // In the nudging process during model quantization, we force // real value of 0.0 be represented by a quantized value. This - // guarantees that the input_offset is a int8, even though it - // is represented using int32. - // int32 += int8 * (int8 - int8) so the highest value we can - // get from each accumulation is [-127, 127] * ([-128, 127] - + // guarantees that the input_offset is a int8_t, even though + // it is represented using int32_t. int32_t += int8_t * + // (int8_t - int8_t) so the highest value we can get from each + // accumulation is [-127, 127] * ([-128, 127] - // [-128, 127]), which is [-32512, 32512]. log2(32512) // = 14.98, which means we can accumulate at least 2^16 // multiplications without overflow. The accumulator is @@ -120,12 +120,12 @@ inline void DepthwiseConvPerChannel( } inline void DepthwiseConvPerChannel( - const DepthwiseParams& params, const int32* output_multiplier, - const int32* output_shift, const RuntimeShape& input_shape, - const int16* input_data, const RuntimeShape& filter_shape, - const int8* filter_data, const RuntimeShape& bias_shape, + const DepthwiseParams& params, const int32_t* output_multiplier, + const int32_t* output_shift, const RuntimeShape& input_shape, + const int16_t* input_data, const RuntimeShape& filter_shape, + const int8_t* filter_data, const RuntimeShape& bias_shape, const std::int64_t* bias_data, const RuntimeShape& output_shape, - int16* output_data) { + int16_t* output_data) { // Get parameters. const int stride_width = params.stride_width; const int stride_height = params.stride_height; @@ -134,8 +134,8 @@ inline void DepthwiseConvPerChannel( const int pad_width = params.padding_values.width; const int pad_height = params.padding_values.height; const int depth_multiplier = params.depth_multiplier; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; // Check dimensions of the tensors. TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); @@ -174,9 +174,9 @@ inline void DepthwiseConvPerChannel( (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && (in_y < input_height); if (is_point_inside_image) { - int32 input_val = input_data[Offset(input_shape, batch, in_y, - in_x, in_channel)]; - int32 filter_val = filter_data[Offset( + int32_t input_val = input_data[Offset( + input_shape, batch, in_y, in_x, in_channel)]; + int32_t filter_val = filter_data[Offset( filter_shape, 0, filter_y, filter_x, output_channel)]; // Accumulate with 64 bits accumulator. // We assume maximum of 2^16 accumulations as with the 8-bit @@ -190,7 +190,7 @@ inline void DepthwiseConvPerChannel( if (bias_data) { acc += bias_data[output_channel]; } - int32 scaled_acc = MultiplyByQuantizedMultiplier( + int32_t scaled_acc = MultiplyByQuantizedMultiplier( acc, output_multiplier[output_channel], output_shift[output_channel]); scaled_acc = std::max(scaled_acc, output_activation_min); @@ -207,8 +207,8 @@ inline void DepthwiseConvPerChannel( inline void DepthwiseConvHybridPerChannel( const DepthwiseParams& params, float* scaling_factors_ptr, - const RuntimeShape& input_shape, const int8* input_data, - const RuntimeShape& filter_shape, const int8* filter_data, + const RuntimeShape& input_shape, const int8_t* input_data, + const RuntimeShape& filter_shape, const int8_t* filter_data, const RuntimeShape& bias_shape, const float* bias_data, const RuntimeShape& output_shape, float* output_data, const float* per_channel_scale, int32_t* input_offset) { @@ -247,7 +247,7 @@ inline void DepthwiseConvHybridPerChannel( const int output_channel = m + in_channel * depth_multiplier; const int in_x_origin = (out_x * stride_width) - pad_width; const int in_y_origin = (out_y * stride_height) - pad_height; - int32 acc = 0; + int32_t acc = 0; for (int filter_y = 0; filter_y < filter_height; ++filter_y) { for (int filter_x = 0; filter_x < filter_width; ++filter_x) { const int in_x = in_x_origin + dilation_width_factor * filter_x; @@ -258,9 +258,9 @@ inline void DepthwiseConvHybridPerChannel( (in_x >= 0) && (in_x < input_width) && (in_y >= 0) && (in_y < input_height); if (is_point_inside_image) { - int32 input_val = input_data[Offset(input_shape, batch, in_y, - in_x, in_channel)]; - int32 filter_val = filter_data[Offset( + int32_t input_val = input_data[Offset( + input_shape, batch, in_y, in_x, in_channel)]; + int32_t filter_val = filter_data[Offset( filter_shape, 0, filter_y, filter_x, output_channel)]; acc += filter_val * (input_val - input_offset[batch]); } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h index fd9cb018..2bc3e794 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h @@ -24,15 +24,15 @@ inline void FullyConnected( const FullyConnectedParams& params, const RuntimeShape& input_shape, const int8_t* input_data, const RuntimeShape& filter_shape, const int8_t* filter_data, const RuntimeShape& bias_shape, - const int32* bias_data, const RuntimeShape& output_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, int8_t* output_data) { - const int32 input_offset = params.input_offset; - const int32 filter_offset = params.weights_offset; - const int32 output_offset = params.output_offset; - const int32 output_multiplier = params.output_multiplier; + const int32_t input_offset = params.input_offset; + const int32_t filter_offset = params.weights_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_multiplier = params.output_multiplier; const int output_shift = params.output_shift; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; TFLITE_DCHECK_GE(filter_shape.DimensionsCount(), 2); TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 2); @@ -44,10 +44,10 @@ inline void FullyConnected( const int accum_depth = filter_shape.Dims(filter_dim_count - 1); for (int b = 0; b < batches; ++b) { for (int out_c = 0; out_c < output_depth; ++out_c) { - int32 acc = 0; + int32_t acc = 0; for (int d = 0; d < accum_depth; ++d) { - int32 input_val = input_data[b * accum_depth + d]; - int32 filter_val = filter_data[out_c * accum_depth + d]; + int32_t input_val = input_data[b * accum_depth + d]; + int32_t filter_val = filter_data[out_c * accum_depth + d]; acc += (filter_val + filter_offset) * (input_val + input_offset); } if (bias_data) { @@ -68,11 +68,11 @@ inline void FullyConnected( const int8_t* filter_data, const RuntimeShape& bias_shape, const int64_t* bias_data, const RuntimeShape& output_shape, int16_t* output_data) { - const int32 filter_offset = params.weights_offset; - const int32 output_multiplier = params.output_multiplier; + const int32_t filter_offset = params.weights_offset; + const int32_t output_multiplier = params.output_multiplier; const int output_shift = params.output_shift; - const int32 output_activation_min = params.quantized_activation_min; - const int32 output_activation_max = params.quantized_activation_max; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; TFLITE_DCHECK_GE(filter_shape.DimensionsCount(), 2); TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 2); @@ -86,8 +86,8 @@ inline void FullyConnected( for (int out_c = 0; out_c < output_depth; ++out_c) { int64_t acc = 0; for (int d = 0; d < accum_depth; ++d) { - int32 input_val = input_data[b * accum_depth + d]; - int32 filter_val = filter_data[out_c * accum_depth + d]; + int32_t input_val = input_data[b * accum_depth + d]; + int32_t filter_val = filter_data[out_c * accum_depth + d]; acc += (filter_val + filter_offset) * input_val; } if (bias_data) { diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h index 7488a214..31f2de98 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h @@ -21,8 +21,8 @@ namespace tflite { namespace reference_integer_ops { inline void L2Normalization(int32_t input_zero_point, int32_t outer_size, - int32_t depth, const int8* input_data, - int8* output_data) { + int32_t depth, const int8_t* input_data, + int8_t* output_data) { static constexpr int8_t kMinInt8 = std::numeric_limits::min(); static constexpr int8_t kMaxInt8 = std::numeric_limits::max(); // The output scale must be in sync with Prepare(). @@ -30,7 +30,7 @@ inline void L2Normalization(int32_t input_zero_point, int32_t outer_size, // to [-1, 127/128]. static constexpr int32_t kOutputScale = 7; for (int outer_index = 0; outer_index < outer_size; ++outer_index) { - // int32 = (int8 - int8) ^ 2. + // int32_t = (int8_t - int8_t) ^ 2. // ([-128, 127] - [-128, 127]) ^ 2 = [0, (2^8 - 1)^2] so the accumulator is // safe from overflowing in at least 2^16 steps. int32_t acc = 0; @@ -55,7 +55,7 @@ inline void L2Normalization(int32_t input_zero_point, int32_t outer_size, std::min(static_cast(kMaxInt8), std::max(static_cast(kMinInt8), output_in_q24)); output_data[depth * outer_index + inner_index] = - static_cast(output_in_q24); + static_cast(output_in_q24); } } } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h index aa626f43..d1a15bd9 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h @@ -58,12 +58,15 @@ inline void Logistic(int32_t input_zero_point, int32_t input_range_radius, } } -inline void Logistic(int32_t input_size, const int16_t* ptr_input_data, - int16_t* ptr_output_data) { +inline void Logistic(int32_t input_multiplier, int32_t input_size, + const int16_t* ptr_input_data, int16_t* ptr_output_data) { // We use the LUT for sigmoid and take into account, that // tanh(x) = 2*sigmoid(2*x) - 1 + + int32_t input_data_mul = (input_multiplier > 0) ? input_multiplier : 1; + for (int i = 0; i < input_size; ++i, ptr_input_data++, ptr_output_data++) { - int32_t input_data = *ptr_input_data; + int32_t input_data = (*ptr_input_data) * input_data_mul; // Scale by 3/4 to expand range [-8,8]->[-10.7,10.7] and // we do interpolation on unsigned values. @@ -72,13 +75,20 @@ inline void Logistic(int32_t input_size, const int16_t* ptr_input_data, // We divide by 2 power of 9, because // we need to divide by 2 in power of 7 for // the input conversion + 1/4 from the scale above. - uint8_t uh = abs_input_data >> 9; - uint32_t ua = sigmoid_table_uint16[uh]; - uint32_t ub = sigmoid_table_uint16[uh + 1]; - uint32_t ut = abs_input_data & 0x1ff; + // Define uh as uint32_t type not to make this function overflow. + uint32_t uh = abs_input_data >> 9; + uint32_t result; - // Interpolation is done using the fractional bit. - uint32_t result = (ua << 9) + ut * (ub - ua); + if (uh >= 255) { + // Saturate to maximum. + result = 0x7FFF << 10; + } else { + uint32_t ua = sigmoid_table_uint16[uh]; + uint32_t ub = sigmoid_table_uint16[uh + 1]; + uint32_t ut = abs_input_data & 0x1ff; + // Interpolation is done using the fractional bit. + result = (ua << 9) + ut * (ub - ua); + } result = (input_data >= 0) ? (result + (1 << 9)) : ((1 << (16 + 9)) - result + (1 << 9) - 1); diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/mean.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/mean.h new file mode 100644 index 00000000..bd484270 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/mean.h @@ -0,0 +1,77 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_MEAN_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_MEAN_H_ + +#include "tensorflow/lite/kernels/internal/common.h" + +namespace tflite { +namespace reference_integer_ops { + +template +inline void Mean(const tflite::MeanParams& op_params, int32_t multiplier, + int32_t shift, const RuntimeShape& unextended_input_shape, + const integer_type* input_data, int32_t input_zero_point, + const RuntimeShape& unextended_output_shape, + integer_type* output_data, int32_t output_zero_point) { + // Current implementation only supports dimension equals 4 and simultaneous + // reduction over width and height. + TFLITE_CHECK_EQ(unextended_input_shape.DimensionsCount(), 4); + TFLITE_CHECK_LE(unextended_output_shape.DimensionsCount(), 4); + const RuntimeShape input_shape = + RuntimeShape::ExtendedShape(4, unextended_input_shape); + const RuntimeShape output_shape = + RuntimeShape::ExtendedShape(4, unextended_output_shape); + const int output_batch = output_shape.Dims(0); + const int output_height = output_shape.Dims(1); + const int output_width = output_shape.Dims(2); + const int output_depth = output_shape.Dims(3); + const int input_height = input_shape.Dims(1); + const int input_width = input_shape.Dims(2); + const int num_elements_in_axis = input_width * input_height; + + TFLITE_CHECK_EQ(op_params.axis_count, 2); + TFLITE_CHECK((op_params.axis[0] == 1 && op_params.axis[1] == 2) || + (op_params.axis[0] == 2 && op_params.axis[1] == 1)); + TFLITE_CHECK_EQ(output_height, 1); + TFLITE_CHECK_EQ(output_width, 1); + + static constexpr int32_t kMinInt = std::numeric_limits::min(); + static constexpr int32_t kMaxInt = std::numeric_limits::max(); + + for (int out_b = 0; out_b < output_batch; ++out_b) { + for (int out_d = 0; out_d < output_depth; ++out_d) { + int32_t acc = 0; + for (int in_h = 0; in_h < input_height; ++in_h) { + for (int in_w = 0; in_w < input_width; ++in_w) { + acc += input_data[Offset(input_shape, out_b, in_h, in_w, out_d)] - + input_zero_point; + } + } + acc = MultiplyByQuantizedMultiplier(acc, multiplier, shift); + acc = acc > 0 ? (acc + num_elements_in_axis / 2) / num_elements_in_axis + : (acc - num_elements_in_axis / 2) / num_elements_in_axis; + acc += output_zero_point; + acc = std::min(std::max(acc, kMinInt), kMaxInt); + output_data[Offset(output_shape, out_b, 0, 0, out_d)] = + static_cast(acc); + } + } +} + +} // namespace reference_integer_ops +} // namespace tflite + +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_MEAN_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/mul.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/mul.h index a815c3f5..b80838aa 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/mul.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/mul.h @@ -27,14 +27,14 @@ inline void MulElementwise(int size, const ArithmeticParams& params, const T* input1_data, const T* input2_data, T* output_data) { for (int i = 0; i < size; ++i) { - const int32 input1_val = params.input1_offset + input1_data[i]; - const int32 input2_val = params.input2_offset + input2_data[i]; - const int32 unclamped_result = + const int32_t input1_val = params.input1_offset + input1_data[i]; + const int32_t input2_val = params.input2_offset + input2_data[i]; + const int32_t unclamped_result = params.output_offset + MultiplyByQuantizedMultiplier(input1_val * input2_val, params.output_multiplier, params.output_shift); - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, unclamped_result)); output_data[i] = static_cast(clamped_output); @@ -57,13 +57,13 @@ inline void Mul(const ArithmeticParams& params, // Mul with 16 bit inputs and int8_t outputs. inline void Mul(const ArithmeticParams& params, - const RuntimeShape& input1_shape, const int16* input1_data, - const RuntimeShape& input2_shape, const int16* input2_data, + const RuntimeShape& input1_shape, const int16_t* input1_data, + const RuntimeShape& input2_shape, const int16_t* input2_data, const RuntimeShape& output_shape, int8_t* output_data) { ruy::profiler::ScopeLabel label("Mul/Int16Int8"); - int32 output_offset = params.output_offset; - int32 output_activation_min = params.quantized_activation_min; - int32 output_activation_max = params.quantized_activation_max; + int32_t output_offset = params.output_offset; + int32_t output_activation_min = params.quantized_activation_min; + int32_t output_activation_max = params.quantized_activation_max; TFLITE_DCHECK_LE(output_activation_min, output_activation_max); const int flat_size = @@ -75,12 +75,12 @@ inline void Mul(const ArithmeticParams& params, F0 unclamped_result = F0::FromRaw(input1_data[i]) * F0::FromRaw(input2_data[i]); - int16 rescaled_result = + int16_t rescaled_result = gemmlowp::RoundingDivideByPOT(unclamped_result.raw(), 8); - int16 clamped_result = - std::min(output_activation_max - output_offset, rescaled_result); - clamped_result = - std::max(output_activation_min - output_offset, clamped_result); + int16_t clamped_result = std::min( + output_activation_max - output_offset, rescaled_result); + clamped_result = std::max(output_activation_min - output_offset, + clamped_result); output_data[i] = output_offset + clamped_result; } } @@ -104,18 +104,18 @@ inline void BroadcastMul4DSlow( for (int y = 0; y < extended_output_shape.Dims(1); ++y) { for (int x = 0; x < extended_output_shape.Dims(2); ++x) { for (int c = 0; c < extended_output_shape.Dims(3); ++c) { - const int32 input1_val = + const int32_t input1_val = params.input1_offset + input1_data[SubscriptToIndex(desc1, b, y, x, c)]; - const int32 input2_val = + const int32_t input2_val = params.input2_offset + input2_data[SubscriptToIndex(desc2, b, y, x, c)]; - const int32 unclamped_result = + const int32_t unclamped_result = params.output_offset + MultiplyByQuantizedMultiplier(input1_val * input2_val, params.output_multiplier, params.output_shift); - const int32 clamped_output = std::min( + const int32_t clamped_output = std::min( params.quantized_activation_max, std::max(params.quantized_activation_min, unclamped_result)); output_data[Offset(extended_output_shape, b, y, x, c)] = diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h index 6b49d2b1..17944bc4 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h @@ -22,8 +22,9 @@ namespace tflite { namespace reference_integer_ops { inline void AveragePool(const PoolParams& params, - const RuntimeShape& input_shape, const int8* input_data, - const RuntimeShape& output_shape, int8* output_data) { + const RuntimeShape& input_shape, + const int8_t* input_data, + const RuntimeShape& output_shape, int8_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); @@ -52,7 +53,7 @@ inline void AveragePool(const PoolParams& params, const int filter_y_start = std::max(0, -in_y_origin); const int filter_y_end = std::min(params.filter_height, input_height - in_y_origin); - int32 acc = 0; + int32_t acc = 0; int filter_count = 0; for (int filter_y = filter_y_start; filter_y < filter_y_end; ++filter_y) { @@ -71,7 +72,7 @@ inline void AveragePool(const PoolParams& params, acc = std::max(acc, params.quantized_activation_min); acc = std::min(acc, params.quantized_activation_max); output_data[Offset(output_shape, batch, out_y, out_x, channel)] = - static_cast(acc); + static_cast(acc); } } } @@ -79,8 +80,8 @@ inline void AveragePool(const PoolParams& params, } inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, - const int8* input_data, const RuntimeShape& output_shape, - int8* output_data) { + const int8_t* input_data, const RuntimeShape& output_shape, + int8_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); TFLITE_DCHECK_GE(params.quantized_activation_min, @@ -137,8 +138,9 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, inline void AveragePool(const PoolParams& params, const RuntimeShape& input_shape, - const int16* input_data, - const RuntimeShape& output_shape, int16* output_data) { + const int16_t* input_data, + const RuntimeShape& output_shape, + int16_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); @@ -167,7 +169,7 @@ inline void AveragePool(const PoolParams& params, const int filter_y_start = std::max(0, -in_y_origin); const int filter_y_end = std::min(params.filter_height, input_height - in_y_origin); - int32 acc = 0; + int32_t acc = 0; int filter_count = 0; for (int filter_y = filter_y_start; filter_y < filter_y_end; ++filter_y) { @@ -186,7 +188,7 @@ inline void AveragePool(const PoolParams& params, acc = std::max(acc, params.quantized_activation_min); acc = std::min(acc, params.quantized_activation_max); output_data[Offset(output_shape, batch, out_y, out_x, channel)] = - static_cast(acc); + static_cast(acc); } } } @@ -194,8 +196,8 @@ inline void AveragePool(const PoolParams& params, } inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, - const int16* input_data, const RuntimeShape& output_shape, - int16* output_data) { + const int16_t* input_data, const RuntimeShape& output_shape, + int16_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); TFLITE_DCHECK_GE(params.quantized_activation_min, diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h new file mode 100644 index 00000000..81ff34fe --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h @@ -0,0 +1,110 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_TANH_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_TANH_H_ + +#include + +#include "fixedpoint/fixedpoint.h" +#include "tensorflow/lite/kernels/internal/common.h" + +namespace tflite { +namespace reference_integer_ops { + +inline void Tanh(int32_t input_zero_point, int32_t input_range_radius, + int32_t input_multiplier, int32_t input_shift, + const RuntimeShape& input_shape, const int8_t* input_data, + const RuntimeShape& output_shape, int8_t* output_data) { + // Integer bits must be in sync with Prepare() function. + static constexpr int32_t kInputIntegerBits = 4; + static constexpr int32_t kOutputScale = 7; + static constexpr int32_t kMinInt8 = std::numeric_limits::min(); + static constexpr int32_t kMaxInt8 = std::numeric_limits::max(); + using F4 = gemmlowp::FixedPoint; + + const int flat_size = MatchingFlatSize(input_shape, output_shape); + + for (int i = 0; i < flat_size; ++i) { + const int32_t input = + static_cast(input_data[i]) - input_zero_point; + if (input <= -input_range_radius) { + output_data[i] = kMinInt8; + } else if (input >= input_range_radius) { + output_data[i] = kMaxInt8; + } else { + const int32_t input_in_q4 = + MultiplyByQuantizedMultiplier(input, input_multiplier, input_shift); + const int32_t output_in_q0 = + gemmlowp::tanh(F4::FromRaw(input_in_q4)).raw(); + + // Rescale and downcast. + using gemmlowp::RoundingDivideByPOT; + int32_t output_in_q24 = + RoundingDivideByPOT(output_in_q0, 31 - kOutputScale); + output_in_q24 = std::min(std::max(output_in_q24, kMinInt8), kMaxInt8); + output_data[i] = static_cast(output_in_q24); + } + } +} + +inline void Tanh(int32_t input_multiplier, int32_t input_left_shift, + const RuntimeShape& input_shape, const int16_t* ptr_input_data, + const RuntimeShape& output_shape, int16_t* ptr_output_data) { + // We use the LUT for sigmoid and take into account, that + // tanh(x) = 2*sigmoid(2*x) - 1 + + int32_t input_data_mul = (input_multiplier > 0) ? input_multiplier : 1; + + int flat_size = MatchingFlatSize(input_shape, output_shape); + + for (int i = 0; i < flat_size; ++i, ptr_input_data++, ptr_output_data++) { + int32_t input_data = (*ptr_input_data) * input_data_mul; + + if (input_left_shift == 1) { + input_data <<= 1; + } + + // Scale by 3/4 to expand range [-8,8]->[-10.7,10.7]. + uint32_t abs_input_data = 3 * abs(input_data); + uint32_t uh = abs_input_data >> 8; + int32_t result; + + if (uh >= 255) { + // Saturate to maximum. + result = 0xFFFF << 8; + } else { + uint32_t ua = sigmoid_table_uint16[uh]; + uint32_t ub = sigmoid_table_uint16[uh + 1]; + + uint8_t ut = abs_input_data & 0xFF; + + result = (ua << 8) + ut * (ub - ua); + } + + result = (input_data >= 0) + ? (result - (1 << (14 + 9)) + (1 << (9 - 2))) + : (-result + (1 << (14 + 9)) + (1 << (9 - 2)) - 1); + + // Convert back to 16-bit. + result >>= (9 - 1); + + *ptr_output_data = result; + } +} + +} // namespace reference_integer_ops +} // namespace tflite + +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_TANH_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/l2normalization.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/l2normalization.h index 00697c2e..7587d2b5 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/l2normalization.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/l2normalization.h @@ -52,40 +52,39 @@ inline void L2Normalization(const tflite::L2NormalizationParams& op_params, inline void L2Normalization(const tflite::L2NormalizationParams& op_params, const RuntimeShape& input_shape, - const uint8* input_data, + const uint8_t* input_data, const RuntimeShape& output_shape, - uint8* output_data) { + uint8_t* output_data) { const int trailing_dim = input_shape.DimensionsCount() - 1; const int depth = MatchingDim(input_shape, trailing_dim, output_shape, trailing_dim); const int outer_size = MatchingFlatSizeSkipDim(input_shape, trailing_dim, output_shape); - const int32 input_zero_point = op_params.input_zero_point; + const int32_t input_zero_point = op_params.input_zero_point; for (int i = 0; i < outer_size; ++i) { - int32 square_l2_norm = 0; + int32_t square_l2_norm = 0; for (int c = 0; c < depth; c++) { - int32 diff = input_data[depth * i + c] - input_zero_point; + int32_t diff = input_data[depth * i + c] - input_zero_point; square_l2_norm += diff * diff; } - int32 inv_l2norm_multiplier; + int32_t inv_l2norm_multiplier; int inv_l2norm_shift; GetInvSqrtQuantizedMultiplierExp(square_l2_norm, kReverseShift, &inv_l2norm_multiplier, &inv_l2norm_shift); for (int c = 0; c < depth; c++) { - int32 diff = input_data[depth * i + c] - input_zero_point; - int32 rescaled_diff = MultiplyByQuantizedMultiplierSmallerThanOneExp( + int32_t diff = input_data[depth * i + c] - input_zero_point; + int32_t rescaled_diff = MultiplyByQuantizedMultiplierSmallerThanOneExp( 128 * diff, inv_l2norm_multiplier, inv_l2norm_shift); - int32 unclamped_output_val = 128 + rescaled_diff; - int32 output_val = - std::min(static_cast(255), - std::max(static_cast(0), unclamped_output_val)); - output_data[depth * i + c] = static_cast(output_val); + int32_t unclamped_output_val = 128 + rescaled_diff; + int32_t output_val = + std::min(static_cast(255), + std::max(static_cast(0), unclamped_output_val)); + output_data[depth * i + c] = static_cast(output_val); } } } - } // namespace reference_ops } // namespace tflite #endif // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_L2NORMALIZATION_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/logistic.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/logistic.h index 8aba5189..64b7133b 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/logistic.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/logistic.h @@ -66,8 +66,8 @@ inline void Logistic(const LogisticParams&, const RuntimeShape& input_shape, } inline void Logistic(const LogisticParams& params, - const RuntimeShape& input_shape, const int16* input_data, - const RuntimeShape& output_shape, int16* output_data) { + const RuntimeShape& input_shape, const int16_t* input_data, + const RuntimeShape& output_shape, int16_t* output_data) { const int flat_size = MatchingFlatSize(input_shape, output_shape); for (int i = 0; i < flat_size; i++) { @@ -84,12 +84,12 @@ inline void Logistic(const LogisticParams& params, } } -// Quantized int8 logistic activation. Cheats by dequantizing and requantizing -// around the floating point logistic method. This implementation is slow on -// platforms without a floating point unit. +// Quantized int8_t logistic activation. Cheats by dequantizing and +// requantizing around the floating point logistic method. This implementation +// is slow on platforms without a floating point unit. -// TODO(b/141211002): Delete this int8 implementation once we can reuse the -// approach used in TFLite for int8 Logistic. +// TODO(b/141211002): Delete this int8_t implementation once we can reuse the +// approach used in TFLite for int8_t Logistic. inline void Logistic(const RuntimeShape& input_shape, const int8_t* input_data, float input_scale, int input_zero_point, const RuntimeShape& output_shape, int8_t* output_data, diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/mul.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/mul.h index 54e947db..0578b81b 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/mul.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/mul.h @@ -24,20 +24,20 @@ namespace reference_ops { // Element-wise mul that can often be used for inner loop of broadcast Mul as // well as the non-broadcast Mul. inline void MulElementwise(int size, const ArithmeticParams& params, - const uint8* input1_data, const uint8* input2_data, - uint8* output_data) { + const uint8_t* input1_data, + const uint8_t* input2_data, uint8_t* output_data) { for (int i = 0; i < size; ++i) { - const int32 input1_val = params.input1_offset + input1_data[i]; - const int32 input2_val = params.input2_offset + input2_data[i]; - const int32 unclamped_result = + const int32_t input1_val = params.input1_offset + input1_data[i]; + const int32_t input2_val = params.input2_offset + input2_data[i]; + const int32_t unclamped_result = params.output_offset + MultiplyByQuantizedMultiplier(input1_val * input2_val, params.output_multiplier, params.output_shift); - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, unclamped_result)); - output_data[i] = static_cast(clamped_output); + output_data[i] = static_cast(clamped_output); } } @@ -60,9 +60,9 @@ inline void Mul(const ArithmeticParams& params, } inline void Mul(const ArithmeticParams& params, - const RuntimeShape& input1_shape, const uint8* input1_data, - const RuntimeShape& input2_shape, const uint8* input2_data, - const RuntimeShape& output_shape, uint8* output_data) { + const RuntimeShape& input1_shape, const uint8_t* input1_data, + const RuntimeShape& input2_shape, const uint8_t* input2_data, + const RuntimeShape& output_shape, uint8_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); const int flat_size = @@ -73,11 +73,11 @@ inline void Mul(const ArithmeticParams& params, inline void BroadcastMul4DSlow(const ArithmeticParams& params, const RuntimeShape& input1_shape, - const uint8* input1_data, + const uint8_t* input1_data, const RuntimeShape& input2_shape, - const uint8* input2_data, + const uint8_t* input2_data, const RuntimeShape& output_shape, - uint8* output_data) { + uint8_t* output_data) { NdArrayDesc<4> desc1; NdArrayDesc<4> desc2; NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1, @@ -89,22 +89,22 @@ inline void BroadcastMul4DSlow(const ArithmeticParams& params, for (int y = 0; y < extended_output_shape.Dims(1); ++y) { for (int x = 0; x < extended_output_shape.Dims(2); ++x) { for (int c = 0; c < extended_output_shape.Dims(3); ++c) { - const int32 input1_val = + const int32_t input1_val = params.input1_offset + input1_data[SubscriptToIndex(desc1, b, y, x, c)]; - const int32 input2_val = + const int32_t input2_val = params.input2_offset + input2_data[SubscriptToIndex(desc2, b, y, x, c)]; - const int32 unclamped_result = + const int32_t unclamped_result = params.output_offset + MultiplyByQuantizedMultiplier(input1_val * input2_val, params.output_multiplier, params.output_shift); - const int32 clamped_output = std::min( + const int32_t clamped_output = std::min( params.quantized_activation_max, std::max(params.quantized_activation_min, unclamped_result)); output_data[Offset(extended_output_shape, b, y, x, c)] = - static_cast(clamped_output); + static_cast(clamped_output); } } } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/pad.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/pad.h index e20aa5e4..2a040cef 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/pad.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/pad.h @@ -32,8 +32,8 @@ constexpr int PadKernelMaxDimensionCount() { return 4; } // equivalent to a simple input1_data. For Pad, it should point to a zero // value. // -// Note that two typenames are required, so that T=P=int32 is considered a -// specialization distinct from P=int32. +// Note that two typenames are required, so that T=P=int32_t is considered a +// specialization distinct from P=int32_t. template inline void PadImpl(const tflite::PadParams& op_params, const RuntimeShape& input_shape, const T* input_data, @@ -116,11 +116,11 @@ inline void Pad(const tflite::PadParams& op_params, output_data); } -// The second (pad-value) input can be int32 when, say, the first is uint8. +// The second (pad-value) input can be int32_t when, say, the first is uint8_t. template inline void Pad(const tflite::PadParams& op_params, const RuntimeShape& input_shape, const T* input_data, - const int32* pad_value_ptr, const RuntimeShape& output_shape, + const int32_t* pad_value_ptr, const RuntimeShape& output_shape, T* output_data) { const T converted_pad_value = static_cast(*pad_value_ptr); PadImpl(op_params, input_shape, input_data, &converted_pad_value, @@ -130,40 +130,18 @@ inline void Pad(const tflite::PadParams& op_params, // This version avoids conflicting template matching. template <> inline void Pad(const tflite::PadParams& op_params, - const RuntimeShape& input_shape, const int32* input_data, - const int32* pad_value_ptr, const RuntimeShape& output_shape, - int32* output_data) { + const RuntimeShape& input_shape, const int32_t* input_data, + const int32_t* pad_value_ptr, const RuntimeShape& output_shape, + int32_t* output_data) { PadImpl(op_params, input_shape, input_data, pad_value_ptr, output_shape, output_data); } -// One could make all PadImageStyle calls simply delegate the work to the -// ordinary Pad. However, it is better that the reference code asserts false in -// similar cases. template inline void PadImageStyle(const tflite::PadParams& op_params, const RuntimeShape& input_shape, const T* input_data, const P* pad_value_ptr, const RuntimeShape& output_shape, T* output_data) { - TFLITE_ASSERT_FALSE; -} - -template -inline void PadImageStyle(const tflite::PadParams& op_params, - const RuntimeShape& input_shape, - const uint8* input_data, const P* pad_value_ptr, - const RuntimeShape& output_shape, - uint8* output_data) { - Pad(op_params, input_shape, input_data, pad_value_ptr, output_shape, - output_data); -} - -template -inline void PadImageStyle(const tflite::PadParams& op_params, - const RuntimeShape& input_shape, - const int8_t* input_data, const P* pad_value_ptr, - const RuntimeShape& output_shape, - int8_t* output_data) { Pad(op_params, input_shape, input_data, pad_value_ptr, output_shape, output_data); } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/pooling.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/pooling.h index a03359cd..0872f521 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/pooling.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/pooling.h @@ -78,8 +78,9 @@ inline void AveragePool(const PoolParams& params, inline void AveragePool(const PoolParams& params, const RuntimeShape& input_shape, - const uint8* input_data, - const RuntimeShape& output_shape, uint8* output_data) { + const uint8_t* input_data, + const RuntimeShape& output_shape, + uint8_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); @@ -108,7 +109,7 @@ inline void AveragePool(const PoolParams& params, const int filter_y_start = std::max(0, -in_y_origin); const int filter_y_end = std::min(params.filter_height, input_height - in_y_origin); - int32 acc = 0; + int32_t acc = 0; int filter_count = 0; for (int filter_y = filter_y_start; filter_y < filter_y_end; ++filter_y) { @@ -125,7 +126,7 @@ inline void AveragePool(const PoolParams& params, acc = std::max(acc, params.quantized_activation_min); acc = std::min(acc, params.quantized_activation_max); output_data[Offset(output_shape, batch, out_y, out_x, channel)] = - static_cast(acc); + static_cast(acc); } } } @@ -237,8 +238,8 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, } inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, - const uint8* input_data, const RuntimeShape& output_shape, - uint8* output_data) { + const uint8_t* input_data, const RuntimeShape& output_shape, + uint8_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); TFLITE_DCHECK_GE(params.quantized_activation_min, 0); @@ -269,7 +270,7 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, const int filter_y_start = std::max(0, -in_y_origin); const int filter_y_end = std::min(params.filter_height, input_height - in_y_origin); - uint8 max = 0; + uint8_t max = 0; for (int filter_y = filter_y_start; filter_y < filter_y_end; ++filter_y) { for (int filter_x = filter_x_start; filter_x < filter_x_end; @@ -281,10 +282,10 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, input_data[Offset(input_shape, batch, in_y, in_x, channel)]); } } - max = std::max(max, params.quantized_activation_min); - max = std::min(max, params.quantized_activation_max); + max = std::max(max, params.quantized_activation_min); + max = std::min(max, params.quantized_activation_max); output_data[Offset(output_shape, batch, out_y, out_x, channel)] = - static_cast(max); + static_cast(max); } } } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/prelu.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/prelu.h index d3d7d78a..02db5174 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/prelu.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/prelu.h @@ -23,7 +23,7 @@ namespace tflite { namespace reference_ops { -// Broadcast prelu to output_shape for quantized uint8/int8 data. +// Broadcast prelu to output_shape for quantized uint8_t/int8_t data. template inline void BroadcastPrelu4DSlow( const PreluParams& params, const RuntimeShape& input_shape, @@ -44,24 +44,26 @@ inline void BroadcastPrelu4DSlow( for (int c = 0; c < extended_output_shape.Dims(3); ++c) { int output_index = Offset(extended_output_shape, b, y, x, c); int input_index = SubscriptToIndex(desc1, b, y, x, c); - const int32 input_value = + const int32_t input_value = params.input_offset + input_data[input_index]; - int32 output_value; + int32_t output_value; if (input_value >= 0) { - output_value = input_value; + output_value = MultiplyByQuantizedMultiplier( + input_value, params.output_multiplier_1, params.output_shift_1); } else { auto alpha_index = SubscriptToIndex(desc2, b, y, x, c); - const int32 alpha_value = + const int32_t alpha_value = params.alpha_offset + alpha_data[alpha_index]; + output_value = MultiplyByQuantizedMultiplier( - input_value * alpha_value, params.output_multiplier, - params.output_shift); + input_value * alpha_value, params.output_multiplier_2, + params.output_shift_2); } output_value += params.output_offset; - const int32 quantized_min = std::numeric_limits::min(); - const int32 quantized_max = std::numeric_limits::max(); - const int32 clamped_output = + const int32_t quantized_min = std::numeric_limits::min(); + const int32_t quantized_max = std::numeric_limits::max(); + const int32_t clamped_output = std::min(quantized_max, std::max(quantized_min, output_value)); output_data[output_index] = static_cast(clamped_output); } @@ -70,6 +72,37 @@ inline void BroadcastPrelu4DSlow( } } +template +inline void Prelu(const PreluParams& params, const RuntimeShape& input_shape, + const T* input_data, const RuntimeShape& alpha_shape, + const T* alpha_data, const RuntimeShape& output_shape, + T* output_data) { + const int32_t quantized_min = std::numeric_limits::min(); + const int32_t quantized_max = std::numeric_limits::max(); + + const int flat_size = + MatchingElementsSize(input_shape, alpha_shape, output_shape); + for (int i = 0; i < flat_size; ++i) { + const int32_t input_value = params.input_offset + input_data[i]; + int32_t output_value; + if (input_value >= 0) { + output_value = MultiplyByQuantizedMultiplier( + input_value, params.output_multiplier_1, params.output_shift_1); + } else { + const int32_t alpha_value = params.alpha_offset + alpha_data[i]; + + output_value = MultiplyByQuantizedMultiplier(input_value * alpha_value, + params.output_multiplier_2, + params.output_shift_2); + } + output_value += params.output_offset; + + const int32_t clamped_output = + std::min(quantized_max, std::max(quantized_min, output_value)); + output_data[i] = static_cast(clamped_output); + } +} + } // namespace reference_ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h index 8e1a6c85..40f779c5 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h @@ -76,6 +76,10 @@ inline bool ProcessBroadcastShapes(const RuntimeShape& shape0, BroadcastableOpCategory::kFirstInputBroadcastsFast && params->broadcast_category != BroadcastableOpCategory::kSecondInputBroadcastsFast) { + // This is unreachable because at least one else clause in the above loop + // must be reached. + TFLITE_DCHECK(false); + params->broadcast_category = BroadcastableOpCategory::kNonBroadcast; return false; } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/quantize.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/quantize.h index 58d19c0a..6f3f9aeb 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/quantize.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/quantize.h @@ -15,7 +15,11 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_QUANTIZE_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_QUANTIZE_H_ +#include +#include + #include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/internal/cppmath.h" #include "tensorflow/lite/kernels/internal/types.h" @@ -29,18 +33,18 @@ inline void AffineQuantize(const tflite::QuantizationParams& op_params, const InputT* input_data, const RuntimeShape& output_shape, OutputT* output_data) { - const int32 zero_point = op_params.zero_point; + const int32_t zero_point = op_params.zero_point; const double scale = op_params.scale; const int flat_size = MatchingFlatSize(input_shape, output_shape); - static constexpr int32 min_val = std::numeric_limits::min(); - static constexpr int32 max_val = std::numeric_limits::max(); + static constexpr int32_t min_val = std::numeric_limits::min(); + static constexpr int32_t max_val = std::numeric_limits::max(); for (int i = 0; i < flat_size; i++) { const InputT val = input_data[i]; - int32 unclamped = - static_cast(TfLiteRound(val / static_cast(scale))) + + int32_t unclamped = + static_cast(TfLiteRound(val / static_cast(scale))) + zero_point; - int32 clamped = std::min(std::max(unclamped, min_val), max_val); + int32_t clamped = std::min(std::max(unclamped, min_val), max_val); output_data[i] = clamped; } } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/reduce.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/reduce.h index 17dfd855..a7c86ddb 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/reduce.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/reduce.h @@ -18,6 +18,8 @@ limitations under the License. #include "ruy/profiler/instrumentation.h" // from @ruy #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/cppmath.h" +#include "tensorflow/lite/kernels/internal/max.h" +#include "tensorflow/lite/kernels/internal/min.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" #include "tensorflow/lite/kernels/internal/types.h" @@ -68,6 +70,9 @@ inline bool ResolveAxis(const int num_dims, const int* axis, // eg: For num_dims=3, [0, 1, 2] is the same as [-3, -2, -1] */ int current = axis[idx] < 0 ? (axis[idx] + num_dims) : axis[idx]; TFLITE_DCHECK(current >= 0 && current < num_dims); + if (current < 0 || current >= num_dims) { + return false; + } bool is_dup = false; for (int j = 0; j < *out_num_axis; ++j) { if (out_axis[j] == current) { @@ -127,6 +132,11 @@ inline bool ReduceGeneric(const T* input_data, const int* input_dims, bool keep_dims, int* temp_index, int* resolved_axis, T init_value, T reducer(const T current, const T in)) { + // Return early when input shape has zero dim. + for (int i = 0; i < input_num_dims; ++i) { + if (input_dims[i] == 0) return true; + } + // Reset output data. if (!InitTensorDataForReduce(output_dims, output_num_dims, init_value, output_data)) { @@ -184,11 +194,11 @@ inline bool Mean(const T* input_data, const int* input_dims, } // Calculate mean by dividing output_data by num of aggregated element. - U num_elements_in_axis = 1; + size_t num_elements_in_axis = 1; for (int idx = 0; idx < num_resolved_axis; ++idx) { size_t current = static_cast(input_dims[resolved_axis[idx]]); // Overflow prevention. - if (current > (std::numeric_limits::max() / num_elements_in_axis)) { + if (current > (std::numeric_limits::max() / num_elements_in_axis)) { return false; } num_elements_in_axis *= current; @@ -249,9 +259,9 @@ inline void Mean(const tflite::MeanParams& op_params, inline void Mean(const tflite::MeanParams& op_params, const RuntimeShape& unextended_input_shape, - const uint8_t* input_data, int32 input_zero_point, + const uint8_t* input_data, int32_t input_zero_point, float input_scale, const RuntimeShape& unextended_output_shape, - uint8_t* output_data, int32 output_zero_point, + uint8_t* output_data, int32_t output_zero_point, float output_scale) { ruy::profiler::ScopeLabel label("Mean4D/Uint8"); @@ -280,9 +290,9 @@ inline void Mean(const tflite::MeanParams& op_params, constexpr int32_t kMinValue = std::numeric_limits::min(); constexpr int32_t kMaxValue = std::numeric_limits::max(); - int32 bias = + int32_t bias = output_zero_point - - static_cast(input_zero_point * input_scale / output_scale); + static_cast(input_zero_point * input_scale / output_scale); double real_scale = static_cast(input_scale / (num_elements_in_axis * output_scale)); @@ -291,7 +301,7 @@ inline void Mean(const tflite::MeanParams& op_params, QuantizeMultiplier(real_scale, &multiplier, &shift); for (int out_b = 0; out_b < output_batch; ++out_b) { for (int out_d = 0; out_d < output_depth; ++out_d) { - int32 acc = 0; + int32_t acc = 0; for (int in_h = 0; in_h < input_height; ++in_h) { for (int in_w = 0; in_w < input_width; ++in_w) { acc += input_data[Offset(input_shape, out_b, in_h, in_w, out_d)]; @@ -310,18 +320,21 @@ inline void Mean(const tflite::MeanParams& op_params, // It does so in two stages, first calculates the sum of elements along the axis // then divides it by the number of element in axis for quantized values. template -inline bool QuantizedMeanOrSum(const T* input_data, int32 input_zero_point, +inline bool QuantizedMeanOrSum(const T* input_data, int32_t input_zero_point, float input_scale, const int* input_dims, const int input_num_dims, T* output_data, - int32 output_zero_point, float output_scale, + int32_t output_zero_point, float output_scale, const int* output_dims, const int output_num_dims, const int* axis, const int num_axis_dimensions, bool keep_dims, int* temp_index, int* resolved_axis, U* temp_sum, bool compute_sum) { - const bool uint8_case = std::is_same::value; + const bool uint8_case = std::is_same::value; + const bool int16_case = std::is_same::value; if (uint8_case) { ruy::profiler::ScopeLabel label(compute_sum ? "Sum/Uint8" : "Mean/Uint8"); + } else if (int16_case) { + ruy::profiler::ScopeLabel label(compute_sum ? "Sum/Int16" : "Mean/Int16"); } else { ruy::profiler::ScopeLabel label(compute_sum ? "Sum/Int8" : "Mean/Int8"); } @@ -354,11 +367,11 @@ inline bool QuantizedMeanOrSum(const T* input_data, int32 input_zero_point, } // Calculate mean by dividing output_data by num of aggregated element. - U num_elements_in_axis = 1; + size_t num_elements_in_axis = 1; for (int idx = 0; idx < num_resolved_axis; ++idx) { size_t current = static_cast(input_dims[resolved_axis[idx]]); // Overflow prevention. - if (current > (std::numeric_limits::max() / num_elements_in_axis)) { + if (current > (std::numeric_limits::max() / num_elements_in_axis)) { return false; } num_elements_in_axis *= current; @@ -368,8 +381,7 @@ inline bool QuantizedMeanOrSum(const T* input_data, int32 input_zero_point, const float scale = input_scale / output_scale; if (compute_sum) { // TODO(b/116341117): Eliminate float and do this completely in 8bit. - const float bias = - -input_zero_point * scale * num_elements_in_axis + 0.5f; + const float bias = -input_zero_point * scale * num_elements_in_axis; for (size_t idx = 0; idx < num_outputs; ++idx) { const U value = static_cast(TfLiteRound(temp_sum[idx] * scale + bias)) + @@ -377,15 +389,15 @@ inline bool QuantizedMeanOrSum(const T* input_data, int32 input_zero_point, output_data[idx] = static_cast(value); } } else { - const float bias = -input_zero_point * scale + 0.5f; + const float bias = -input_zero_point * scale; for (size_t idx = 0; idx < num_outputs; ++idx) { float float_mean = static_cast(temp_sum[idx]) / static_cast(num_elements_in_axis); - float result = - std::min(TfLiteRound(float_mean * scale + bias) + output_zero_point, - static_cast(std::numeric_limits::max())); - result = - std::max(result, static_cast(std::numeric_limits::min())); + float result = TfLiteMin( + TfLiteRound(float_mean * scale + bias) + output_zero_point, + static_cast(std::numeric_limits::max())); + result = TfLiteMax(result, + static_cast(std::numeric_limits::min())); output_data[idx] = static_cast(result); } } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h index ed87863a..95550abc 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h @@ -17,28 +17,30 @@ limitations under the License. #include +#include "tensorflow/lite/kernels/internal/cppmath.h" #include "tensorflow/lite/kernels/internal/types.h" namespace tflite { namespace reference_ops { -inline int32 GetNearestNeighbor(const int input_value, const int32 input_size, - const int32 output_size, - const bool align_corners, - const bool half_pixel_centers) { +inline int32_t GetNearestNeighbor(const int input_value, + const int32_t input_size, + const int32_t output_size, + const bool align_corners, + const bool half_pixel_centers) { const float scale = (align_corners && output_size > 1) ? (input_size - 1) / static_cast(output_size - 1) : input_size / static_cast(output_size); const float offset = half_pixel_centers ? 0.5f : 0.0f; - int32 output_value = std::min( + int32_t output_value = std::min( align_corners - ? static_cast(std::round((input_value + offset) * scale)) - : static_cast(std::floor((input_value + offset) * scale)), + ? static_cast(TfLiteRound((input_value + offset) * scale)) + : static_cast(std::floor((input_value + offset) * scale)), input_size - 1); if (half_pixel_centers) { - output_value = std::max(static_cast(0), output_value); + output_value = std::max(static_cast(0), output_value); } return output_value; } @@ -47,7 +49,7 @@ template inline void ResizeNearestNeighbor( const tflite::ResizeNearestNeighborParams& op_params, const RuntimeShape& unextended_input_shape, const T* input_data, - const RuntimeShape& output_size_shape, const int32* output_size_data, + const RuntimeShape& output_size_shape, const int32_t* output_size_data, const RuntimeShape& unextended_output_shape, T* output_data) { TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4); TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4); @@ -57,16 +59,16 @@ inline void ResizeNearestNeighbor( const RuntimeShape output_shape = RuntimeShape::ExtendedShape(4, unextended_output_shape); - int32 batches = MatchingDim(input_shape, 0, output_shape, 0); - int32 input_height = input_shape.Dims(1); - int32 input_width = input_shape.Dims(2); - int32 depth = MatchingDim(input_shape, 3, output_shape, 3); + int32_t batches = MatchingDim(input_shape, 0, output_shape, 0); + int32_t input_height = input_shape.Dims(1); + int32_t input_width = input_shape.Dims(2); + int32_t depth = MatchingDim(input_shape, 3, output_shape, 3); // The Tensorflow version of this op allows resize on the width and height // axis only. TFLITE_DCHECK_EQ(output_size_shape.FlatSize(), 2); - int32 output_height = output_size_data[0]; - int32 output_width = output_size_data[1]; + int32_t output_height = output_size_data[0]; + int32_t output_width = output_size_data[1]; const int col_offset = input_shape.Dims(3); const int row_offset = input_shape.Dims(2) * col_offset; @@ -76,14 +78,14 @@ inline void ResizeNearestNeighbor( T* output_ptr = output_data; for (int b = 0; b < batches; ++b) { for (int y = 0; y < output_height; ++y) { - int32 in_y = GetNearestNeighbor(y, input_height, output_height, - op_params.align_corners, - op_params.half_pixel_centers); - const T* y_input_ptr = input_ptr + in_y * row_offset; - for (int x = 0; x < output_width; ++x) { - int32 in_x = GetNearestNeighbor(x, input_width, output_width, + int32_t in_y = GetNearestNeighbor(y, input_height, output_height, op_params.align_corners, op_params.half_pixel_centers); + const T* y_input_ptr = input_ptr + in_y * row_offset; + for (int x = 0; x < output_width; ++x) { + int32_t in_x = GetNearestNeighbor(x, input_width, output_width, + op_params.align_corners, + op_params.half_pixel_centers); const T* x_input_ptr = y_input_ptr + in_x * col_offset; memcpy(output_ptr, x_input_ptr, depth * sizeof(T)); output_ptr += depth; diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/softmax.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/softmax.h index dd44b3c7..1b3f1181 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/softmax.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/softmax.h @@ -16,7 +16,6 @@ limitations under the License. #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_SOFTMAX_H_ #include -#include #include "fixedpoint/fixedpoint.h" #include "tensorflow/lite/kernels/internal/common.h" @@ -49,26 +48,27 @@ inline void Softmax(const SoftmaxParams& params, // Compute sum. float sum = 0.f; for (int c = 0; c < depth; ++c) { - sum += std::exp((input_data[i * depth + c] - max) * - static_cast(params.beta)); + const float exp_c = std::exp((input_data[i * depth + c] - max) * + static_cast(params.beta)); + output_data[i * depth + c] = exp_c; + sum += exp_c; } // Compute result. for (int c = 0; c < depth; ++c) { - output_data[i * depth + c] = std::exp((input_data[i * depth + c] - max) * - static_cast(params.beta)) / - sum; + output_data[i * depth + c] = output_data[i * depth + c] / sum; } } } -// Quantized softmax with int8/uint8 input and int8/uint8/int16 output. +// Quantized softmax with int8_t/uint8_t input and int8_t/uint8_t/int16_t +// output. template inline void Softmax(const SoftmaxParams& params, const RuntimeShape& input_shape, const InputT* input_data, const RuntimeShape& output_shape, OutputT* output_data) { - const int32 input_beta_multiplier = params.input_multiplier; - const int32 input_beta_left_shift = params.input_left_shift; + const int32_t input_beta_multiplier = params.input_multiplier; + const int32_t input_beta_left_shift = params.input_left_shift; const int diff_min = params.diff_min; // The representation chosen for the input to the exp() function is Q5.26. // We need to leave extra space since values that we skip might be as large as @@ -78,9 +78,10 @@ inline void Softmax(const SoftmaxParams& params, static const int kScaledDiffIntegerBits = 5; static const int kAccumulationIntegerBits = 12; using FixedPointScaledDiff = - gemmlowp::FixedPoint; - using FixedPointAccum = gemmlowp::FixedPoint; - using FixedPoint0 = gemmlowp::FixedPoint; + gemmlowp::FixedPoint; + using FixedPointAccum = + gemmlowp::FixedPoint; + using FixedPoint0 = gemmlowp::FixedPoint; const int trailing_dim = input_shape.DimensionsCount() - 1; const int outer_size = @@ -96,10 +97,10 @@ inline void Softmax(const SoftmaxParams& params, FixedPointAccum sum_of_exps = FixedPointAccum::Zero(); for (int c = 0; c < depth; ++c) { - int32 input_diff = - static_cast(input_data[i * depth + c]) - max_in_row; + int32_t input_diff = + static_cast(input_data[i * depth + c]) - max_in_row; if (input_diff >= diff_min) { - const int32 input_diff_rescaled = + const int32_t input_diff_rescaled = MultiplyByQuantizedMultiplierGreaterThanOne( input_diff, input_beta_multiplier, input_beta_left_shift); const FixedPointScaledDiff scaled_diff_f8 = @@ -114,28 +115,28 @@ inline void Softmax(const SoftmaxParams& params, sum_of_exps.raw(), kAccumulationIntegerBits, &num_bits_over_unit)); for (int c = 0; c < depth; ++c) { - int32 input_diff = - static_cast(input_data[i * depth + c]) - max_in_row; + int32_t input_diff = + static_cast(input_data[i * depth + c]) - max_in_row; if (input_diff >= diff_min) { - const int32 input_diff_rescaled = + const int32_t input_diff_rescaled = MultiplyByQuantizedMultiplierGreaterThanOne( input_diff, input_beta_multiplier, input_beta_left_shift); const FixedPointScaledDiff scaled_diff_f8 = FixedPointScaledDiff::FromRaw(input_diff_rescaled); FixedPoint0 exp_in_0 = exp_on_negative_values(scaled_diff_f8); - int32 unsat_output = gemmlowp::RoundingDivideByPOT( + int32_t unsat_output = gemmlowp::RoundingDivideByPOT( (shifted_scale * exp_in_0).raw(), num_bits_over_unit + 31 - (sizeof(OutputT) * 8)); - const int32 shifted_output = + const int32_t shifted_output = unsat_output + - static_cast(std::numeric_limits::min()); + static_cast(std::numeric_limits::min()); output_data[i * depth + c] = static_cast(std::max( std::min(shifted_output, - static_cast(std::numeric_limits::max())), - static_cast(std::numeric_limits::min()))); + static_cast(std::numeric_limits::max())), + static_cast(std::numeric_limits::min()))); } else { output_data[i * depth + c] = std::numeric_limits::min(); } @@ -143,7 +144,24 @@ inline void Softmax(const SoftmaxParams& params, } } -// Quantized softmax with int16 input and int16 output. +// Computes exp(input - max_input) +inline int16_t SoftMaxCalculateExp(const SoftmaxParams& params, + const int16_t* input_data, const int depth, + int16_t max_in_row, int i, int c) { + int32_t input_diff = input_data[i * depth + c] - max_in_row; + // scale the input_diff such that [-65535, 0] correspond to [-10.0, 0.0] + // exp lut generated with range [-10, 0], as exp(-10) is negligible. + int32_t scaled_diff = MultiplyByQuantizedMultiplier( + input_diff, params.input_multiplier, params.input_left_shift); + // recenter to [-32768, 32767] + int32_t sym_scaled_diff = scaled_diff + 32767; + int16_t sat_sym_scaled_diff = + std::min(std::max(sym_scaled_diff, static_cast(-32768)), + static_cast(32767)); + // apply the exp() LUT activation function + return generic_int16_table_lookup(sat_sym_scaled_diff, params.exp_lut); +} +// Quantized softmax with int16_t input and int16_t output. inline void SoftmaxInt16(const SoftmaxParams& params, const RuntimeShape& input_shape, const int16_t* input_data, @@ -162,28 +180,16 @@ inline void SoftmaxInt16(const SoftmaxParams& params, max_in_row = std::max(max_in_row, input_data[i * depth + c]); } - // Compute exp(input - max_input) - std::vector exp_result_Q015(depth); + // This loops computes the exp values and their sum. We will need the exp + // values later on in the function so we cache them in the output_data + // buffer. This is an optimization done to avoid calculating the exp values + // twice making use of the output_data buffer as scratch memory. + int32_t sum_of_exps = 0; // Q16.15 fixed point format. + int16_t* exp_results_Q015 = output_data + i * depth; for (int c = 0; c < depth; ++c) { - int32_t input_diff = input_data[i * depth + c] - max_in_row; - // scale the input_diff such that [-65535, 0] correspond to [-10.0, 0.0] - int32_t scaled_diff = MultiplyByQuantizedMultiplier( - input_diff, params.input_multiplier, params.input_left_shift); - // recenter to [-32768, 32767] - int32_t sym_scaled_diff = scaled_diff + 32767; - int16_t sat_sym_scaled_diff = - std::min(std::max(sym_scaled_diff, static_cast(-32768)), - static_cast(32767)); - // apply the exp() LUT activation function - exp_result_Q015[c] = - generic_int16_table_lookup(sat_sym_scaled_diff, params.exp_lut); - } - - // sum_of_exps is a Q16.15 fixed point format. - int32_t sum_of_exps = 0; - for (int c = 0; c < depth; ++c) { - // Q16.15 + Q0.15 - sum_of_exps += exp_result_Q015[c]; + exp_results_Q015[c] = + SoftMaxCalculateExp(params, input_data, depth, max_in_row, i, c); + sum_of_exps += exp_results_Q015[c]; } // Compute the reciprocal 1/sum_of_exps @@ -209,7 +215,7 @@ inline void SoftmaxInt16(const SoftmaxParams& params, for (int c = 0; c < depth; ++c) { uint8_t right_shift = 31 - headroom_plus_one; int64_t round = 1 << (right_shift - 1); - int32_t result = (static_cast(exp_result_Q015[c]) * + int32_t result = (static_cast(exp_results_Q015[c]) * static_cast(reciprocal_scale_Q015) + round) >> right_shift; diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/strided_slice.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/strided_slice.h index ba6d4c22..8b6f0c13 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/strided_slice.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/strided_slice.h @@ -16,8 +16,10 @@ limitations under the License. #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_STRIDED_SLICE_H_ #include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/internal/strided_slice_logic.h" #include "tensorflow/lite/kernels/internal/types.h" + namespace tflite { namespace reference_ops { diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/sub.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/sub.h index 48d03de0..b27f251d 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/sub.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/sub.h @@ -15,9 +15,15 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_SUB_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_SUB_H_ -#include "fixedpoint/fixedpoint.h" +#include + +#include +#include + #include "ruy/profiler/instrumentation.h" // from @ruy #include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/kernels/internal/types.h" namespace tflite { @@ -41,11 +47,11 @@ inline void SubNonBroadcast(const ArithmeticParams& params, inline void SubNonBroadcast(const ArithmeticParams& params, const RuntimeShape& input1_shape, - const int32* input1_data, + const int32_t* input1_data, const RuntimeShape& input2_shape, - const int32* input2_data, + const int32_t* input2_data, const RuntimeShape& output_shape, - int32* output_data) { + int32_t* output_data) { const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape); for (int i = 0; i < flat_size; ++i) { @@ -106,12 +112,12 @@ inline void BroadcastSubSlow(const ArithmeticParams& params, template inline void BroadcastSubSlow(const ArithmeticParams& params, const RuntimeShape& input1_shape, - const uint8* input1_data, + const uint8_t* input1_data, const RuntimeShape& input2_shape, - const uint8* input2_data, + const uint8_t* input2_data, const RuntimeShape& output_shape, - uint8* output_data) { - ruy::profiler::ScopeLabel label("BroadcastSubSlow/uint8"); + uint8_t* output_data) { + ruy::profiler::ScopeLabel label("BroadcastSubSlow/uint8_t"); TFLITE_DCHECK_LE(input1_shape.DimensionsCount(), N); TFLITE_DCHECK_LE(input2_shape.DimensionsCount(), N); TFLITE_DCHECK_LE(output_shape.DimensionsCount(), N); @@ -134,28 +140,28 @@ inline void BroadcastSubSlow(const ArithmeticParams& params, // nesting loops such that the innermost loop has the smallest stride for the // best cache behavior. auto sub_func = [&](int indexes[N]) { - const int32 input1_val = + const int32_t input1_val = params.input1_offset + input1_data[SubscriptToIndex(desc1, indexes)]; - const int32 input2_val = + const int32_t input2_val = params.input2_offset + input2_data[SubscriptToIndex(desc2, indexes)]; - const int32 shifted_input1_val = input1_val * (1 << params.left_shift); - const int32 shifted_input2_val = input2_val * (1 << params.left_shift); - const int32 scaled_input1_val = + const int32_t shifted_input1_val = input1_val * (1 << params.left_shift); + const int32_t shifted_input2_val = input2_val * (1 << params.left_shift); + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, params.input1_multiplier, params.input1_shift); - const int32 scaled_input2_val = + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, params.input2_multiplier, params.input2_shift); - const int32 raw_sub = scaled_input1_val - scaled_input2_val; - const int32 raw_output = + const int32_t raw_sub = scaled_input1_val - scaled_input2_val; + const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp( raw_sub, params.output_multiplier, params.output_shift) + params.output_offset; - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); output_data[SubscriptToIndex(output_desc, indexes)] = - static_cast(clamped_output); + static_cast(clamped_output); }; NDOpsHelper(output_desc, sub_func); } @@ -163,12 +169,12 @@ inline void BroadcastSubSlow(const ArithmeticParams& params, template inline void BroadcastSubSlow(const ArithmeticParams& params, const RuntimeShape& input1_shape, - const int32* input1_data, + const int32_t* input1_data, const RuntimeShape& input2_shape, - const int32* input2_data, + const int32_t* input2_data, const RuntimeShape& output_shape, - int32* output_data) { - ruy::profiler::ScopeLabel label("BroadcastSubSlow/int32"); + int32_t* output_data) { + ruy::profiler::ScopeLabel label("BroadcastSubSlow/int32_t"); TFLITE_DCHECK_LE(input1_shape.DimensionsCount(), N); TFLITE_DCHECK_LE(input2_shape.DimensionsCount(), N); TFLITE_DCHECK_LE(output_shape.DimensionsCount(), N); @@ -208,7 +214,7 @@ inline void BroadcastSubSlow(const ArithmeticParams& params, const int8_t* input2_data, const RuntimeShape& output_shape, int8_t* output_data) { - ruy::profiler::ScopeLabel label("BroadcastSubSlow/int8"); + ruy::profiler::ScopeLabel label("BroadcastSubSlow/int8_t"); NdArrayDesc desc1; NdArrayDesc desc2; NdArrayDesc output_desc; @@ -254,6 +260,45 @@ inline void BroadcastSubSlow(const ArithmeticParams& params, NDOpsHelper(output_desc, sub_func); } +template +void BroadcastSubSlow(const ArithmeticParams& params, + const RuntimeShape& input1_shape, + const int64_t* input1_data, + const RuntimeShape& input2_shape, + const int64_t* input2_data, + const RuntimeShape& output_shape, int64_t* output_data) { + ruy::profiler::ScopeLabel label("BroadcastSubSlow/int64_t"); + TFLITE_DCHECK_LE(input1_shape.DimensionsCount(), N); + TFLITE_DCHECK_LE(input2_shape.DimensionsCount(), N); + TFLITE_DCHECK_LE(output_shape.DimensionsCount(), N); + NdArrayDesc desc1; + NdArrayDesc desc2; + NdArrayDesc output_desc; + NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1, + &desc2); + CopyDimsToDesc(RuntimeShape::ExtendedShape(N, output_shape), &output_desc); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest stride, + // typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for the + // best cache behavior. + auto sub_func = [&](int indexes[N]) { + output_data[SubscriptToIndex(output_desc, indexes)] = + ActivationFunctionWithMinMax( + input1_data[SubscriptToIndex(desc1, indexes)] - + input2_data[SubscriptToIndex(desc2, indexes)], + params.int64_activation_min, params.int64_activation_max); + }; + NDOpsHelper(output_desc, sub_func); +} + template void BroadcastSubSlow(const ArithmeticParams& params, const RuntimeShape& input1_shape, const T* input1_data, @@ -294,33 +339,33 @@ void BroadcastSubSlow(const ArithmeticParams& params, // Element-wise Sub that can often be used for inner loop of broadcast sub as // well as the non-broadcast sub. inline void SubElementwise(int size, const ArithmeticParams& params, - const uint8* input1_data, const uint8* input2_data, - uint8* output_data) { + const uint8_t* input1_data, + const uint8_t* input2_data, uint8_t* output_data) { TFLITE_DCHECK_GT(params.input1_offset, -256); TFLITE_DCHECK_GT(params.input2_offset, -256); TFLITE_DCHECK_LT(params.input1_offset, 256); TFLITE_DCHECK_LT(params.input2_offset, 256); for (int i = 0; i < size; ++i) { - const int32 input1_val = params.input1_offset + input1_data[i]; - const int32 input2_val = params.input2_offset + input2_data[i]; - const int32 shifted_input1_val = input1_val * (1 << params.left_shift); - const int32 shifted_input2_val = input2_val * (1 << params.left_shift); - const int32 scaled_input1_val = + const int32_t input1_val = params.input1_offset + input1_data[i]; + const int32_t input2_val = params.input2_offset + input2_data[i]; + const int32_t shifted_input1_val = input1_val * (1 << params.left_shift); + const int32_t shifted_input2_val = input2_val * (1 << params.left_shift); + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, params.input1_multiplier, params.input1_shift); - const int32 scaled_input2_val = + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, params.input2_multiplier, params.input2_shift); - const int32 raw_sub = scaled_input1_val - scaled_input2_val; - const int32 raw_output = + const int32_t raw_sub = scaled_input1_val - scaled_input2_val; + const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp( raw_sub, params.output_multiplier, params.output_shift) + params.output_offset; - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); - output_data[i] = static_cast(clamped_output); + output_data[i] = static_cast(clamped_output); } } @@ -336,22 +381,22 @@ inline void SubElementwise(int size, const ArithmeticParams& params, TFLITE_DCHECK_LE(params.input2_offset, int8_max_value); for (int i = 0; i < size; ++i) { - const int32 input1_val = params.input1_offset + input1_data[i]; - const int32 input2_val = params.input2_offset + input2_data[i]; - const int32 shifted_input1_val = input1_val * (1 << params.left_shift); - const int32 shifted_input2_val = input2_val * (1 << params.left_shift); - const int32 scaled_input1_val = + const int32_t input1_val = params.input1_offset + input1_data[i]; + const int32_t input2_val = params.input2_offset + input2_data[i]; + const int32_t shifted_input1_val = input1_val * (1 << params.left_shift); + const int32_t shifted_input2_val = input2_val * (1 << params.left_shift); + const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input1_val, params.input1_multiplier, params.input1_shift); - const int32 scaled_input2_val = + const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp( shifted_input2_val, params.input2_multiplier, params.input2_shift); - const int32 raw_sub = scaled_input1_val - scaled_input2_val; - const int32 raw_output = + const int32_t raw_sub = scaled_input1_val - scaled_input2_val; + const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp( raw_sub, params.output_multiplier, params.output_shift) + params.output_offset; - const int32 clamped_output = + const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); output_data[i] = static_cast(clamped_output); @@ -359,9 +404,9 @@ inline void SubElementwise(int size, const ArithmeticParams& params, } inline void Sub(const ArithmeticParams& params, - const RuntimeShape& input1_shape, const uint8* input1_data, - const RuntimeShape& input2_shape, const uint8* input2_data, - const RuntimeShape& output_shape, uint8* output_data) { + const RuntimeShape& input1_shape, const uint8_t* input1_data, + const RuntimeShape& input2_shape, const uint8_t* input2_data, + const RuntimeShape& output_shape, uint8_t* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); const int flat_size = @@ -428,40 +473,43 @@ void Sub(const ArithmeticParams& params, const RuntimeShape& input1_shape, } } -inline void SubWithActivation(const ArithmeticParams& params, - const RuntimeShape& input1_shape, - const int32* input1_data, - const RuntimeShape& input2_shape, - const int32* input2_data, - const RuntimeShape& output_shape, - int32* output_data) { +inline void SetActivationMinMax(const ArithmeticParams& params, + int32_t* activation_min, + int32_t* activation_max) { + *activation_min = params.quantized_activation_min; + *activation_max = params.quantized_activation_max; +} + +inline void SetActivationMinMax(const ArithmeticParams& params, + float* activation_min, float* activation_max) { + *activation_min = params.float_activation_min; + *activation_max = params.float_activation_max; +} + +inline void SetActivationMinMax(const ArithmeticParams& params, + int64_t* activation_min, + int64_t* activation_max) { + *activation_min = params.int64_activation_min; + *activation_max = params.int64_activation_max; +} + +template +inline void SubWithActivation( + const ArithmeticParams& params, const RuntimeShape& input1_shape, + const T* input1_data, const RuntimeShape& input2_shape, + const T* input2_data, const RuntimeShape& output_shape, T* output_data) { ruy::profiler::ScopeLabel label("SubWithActivation"); const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape); + T activation_min, activation_max; + SetActivationMinMax(params, &activation_min, &activation_max); + for (int i = 0; i < flat_size; ++i) { output_data[i] = ActivationFunctionWithMinMax( - input1_data[i] - input2_data[i], params.quantized_activation_min, - params.quantized_activation_max); + input1_data[i] - input2_data[i], activation_min, activation_max); } } -inline void SubWithActivation(const ArithmeticParams& params, - const RuntimeShape& input1_shape, - const float* input1_data, - const RuntimeShape& input2_shape, - const float* input2_data, - const RuntimeShape& output_shape, - float* output_data) { - const int flat_size = - MatchingElementsSize(input1_shape, input2_shape, output_shape); - for (int i = 0; i < flat_size; ++i) { - output_data[i] = ActivationFunctionWithMinMax( - input1_data[i] - input2_data[i], params.float_activation_min, - params.float_activation_max); - } -} - - } // namespace reference_ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/tanh.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/tanh.h new file mode 100644 index 00000000..3a05c474 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/reference/tanh.h @@ -0,0 +1,129 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_TANH_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_TANH_H_ + +#include + +#include "fixedpoint/fixedpoint.h" +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/cppmath.h" +#include "tensorflow/lite/kernels/internal/types.h" +#include "tensorflow/lite/kernels/op_macros.h" + +namespace tflite { +namespace reference_ops { + +inline void Tanh(const RuntimeShape& input_shape, const float* input_data, + const RuntimeShape& output_shape, float* output_data) { + const int flat_size = MatchingFlatSize(input_shape, output_shape); + + for (int i = 0; i < flat_size; i++) { + float val = input_data[i]; + float result = std::tanh(val); + output_data[i] = result; + } +} + +// Convenience version that allows, for example, generated-code calls to be +// uniform between data types. +inline void Tanh(const TanhParams&, const RuntimeShape& input_shape, + const float* input_data, const RuntimeShape& output_shape, + float* output_data) { + // Drop params: not needed. + Tanh(input_shape, input_data, output_shape, output_data); +} + +inline void Tanh(const TanhParams& params, const RuntimeShape& input_shape, + const int16_t* input_data, const RuntimeShape& output_shape, + int16_t* output_data) { + const int input_left_shift = params.input_left_shift; + // Support for shifts is limited until we have a parameterized version of + // SaturatingRoundingMultiplyByPOT(). + TFLITE_DCHECK_GE(input_left_shift, 0); + TFLITE_DCHECK_LE(input_left_shift, 1); + + const int flat_size = MatchingFlatSize(input_shape, output_shape); + + // F0 uses 0 integer bits, range [-1, 1]. + // This is the return type of math functions such as tanh, logistic, + // whose range is in [-1, 1]. + using F0 = gemmlowp::FixedPoint; + // F3 uses 3 integer bits, range [-8, 8], the input range expected here. + using F3 = gemmlowp::FixedPoint; + + if (input_left_shift == 0) { + for (int i = 0; i < flat_size; i++) { + F3 input = F3::FromRaw(input_data[i]); + F0 output = gemmlowp::tanh(input); + output_data[i] = output.raw(); + } + } else { + for (int i = 0; i < flat_size; i++) { + F3 input = F3::FromRaw( + gemmlowp::SaturatingRoundingMultiplyByPOT<1>(input_data[i])); + F0 output = gemmlowp::tanh(input); + output_data[i] = output.raw(); + } + } +} + +inline void Tanh(const TanhParams& params, const RuntimeShape& input_shape, + const uint8_t* input_data, const RuntimeShape& output_shape, + uint8_t* output_data) { + const int32_t input_zero_point = params.input_zero_point; + const int32_t input_range_radius = params.input_range_radius; + const int32_t input_multiplier = params.input_multiplier; + const int input_left_shift = params.input_left_shift; + const int32_t output_zero_point = 128; + const int flat_size = MatchingFlatSize(input_shape, output_shape); + + for (int i = 0; i < flat_size; i++) { + const uint8_t input_val_u8 = input_data[i]; + const int32_t input_val_centered = + static_cast(input_val_u8) - input_zero_point; + uint8_t output_val; + if (input_val_centered <= -input_range_radius) { + output_val = 0; + } else if (input_val_centered >= input_range_radius) { + output_val = 255; + } else { + const int32_t input_val_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_val_centered, input_multiplier, input_left_shift); + using FixedPoint4 = gemmlowp::FixedPoint; + using FixedPoint0 = gemmlowp::FixedPoint; + const FixedPoint4 input_val_f4 = FixedPoint4::FromRaw(input_val_rescaled); + const FixedPoint0 output_val_f0 = gemmlowp::tanh(input_val_f4); + // Convert from Q0.31 to Q24.7. + using gemmlowp::RoundingDivideByPOT; + int32_t output_val_s32 = RoundingDivideByPOT(output_val_f0.raw(), 24); + output_val_s32 += output_zero_point; + if (output_val_s32 == 256) { + output_val_s32 = 255; + } + // Reinterpret as Q0.7, encoded in uint8_t. + TFLITE_DCHECK_GE(output_val_s32, 0); + TFLITE_DCHECK_LE(output_val_s32, 255); + output_val = static_cast(output_val_s32); + } + output_data[i] = output_val; + } +} + +} // namespace reference_ops +} // namespace tflite + +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_TANH_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/strided_slice_logic.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/strided_slice_logic.h index d9b5acbb..3d91fbdb 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/strided_slice_logic.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/strided_slice_logic.h @@ -18,6 +18,7 @@ limitations under the License. #include #include + #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/internal/types.h" @@ -69,8 +70,8 @@ inline void StridedSlicePadIndices(tflite::StridedSliceParams* p, } // Return the index for the first element along that axis. This index will be a -// positive integer between [0, axis_size - 1] that can be used to index -// directly into the data. +// positive integer between [0, axis_size] (or [-1, axis_size -1] if stride < 0) +// that can be used to index directly into the data. inline int StartForAxis(const tflite::StridedSliceParams& params, const RuntimeShape& input_shape, int axis) { const auto begin_mask = params.begin_mask; @@ -102,7 +103,13 @@ inline int StartForAxis(const tflite::StridedSliceParams& params, } // Clamping - start = Clamp(start, 0, axis_size - 1); + if (strides[axis] > 0) { + // Forward iteration + start = Clamp(start, 0, axis_size); + } else { + // Backward iteration + start = Clamp(start, -1, axis_size - 1); + } return start; } diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/internal/types.h b/code/lib/tfmicro/tensorflow/lite/kernels/internal/types.h index cbdedd88..37403a88 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/internal/types.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/internal/types.h @@ -24,24 +24,29 @@ limitations under the License. namespace tflite { -enum class FusedActivationFunctionType : uint8 { kNone, kRelu6, kRelu1, kRelu }; -enum class PaddingType : uint8 { kNone, kSame, kValid }; +enum class FusedActivationFunctionType : uint8_t { + kNone, + kRelu6, + kRelu1, + kRelu +}; +enum class PaddingType : uint8_t { kNone, kSame, kValid }; struct PaddingValues { - int16 width; - int16 height; + int16_t width; + int16_t height; // offset is used for calculating "remaining" padding, for example, `width` // is 1 and `width_offset` is 1, so padding_left is 1 while padding_right is // 1 + 1 = 2. - int16 width_offset; + int16_t width_offset; // Same as width_offset except it's over the height dimension. - int16 height_offset; + int16_t height_offset; }; // This enumeration allows for non-default formats for the weights array // of a fully-connected operator, allowing the use of special optimized // runtime paths. -enum class FullyConnectedWeightsFormat : uint8 { +enum class FullyConnectedWeightsFormat : uint8_t { // Default format (flat 2D layout, the inner contiguous dimension // is input_depth, the outer non-contiguous dimension is output_depth) kDefault, @@ -88,11 +93,11 @@ enum class FullyConnectedWeightsFormat : uint8 { // maximize arithmetic throughput. // // Finally, the 'Int8' part in the name refers to the fact that this - // weights format has each weights value encoded as a signed int8 value, - // even if the data type of the weights buffer is uint8. This is intended + // weights format has each weights value encoded as a signed int8_t value, + // even if the data type of the weights buffer is uint8_t. This is intended // to save runtime kernels the effort to have to XOR the top bit of these // bytes before using them in signed arithmetic, see this file for more - // explanations on the 'signed int8 trick' in matrix multiplication kernels: + // explanations on the 'signed int8_t trick' in matrix multiplication kernels: // // tensorflow/lite/toco/graph_transformations/ensure_uint8_weights_safe_for_fast_int8_kernels.cc // @@ -111,7 +116,7 @@ enum class FullyConnectedWeightsFormat : uint8 { // the real 0 value, and scale designates the difference between the real values // corresponding to consecutive quantized values differing by 1. struct QuantizationParams { - int32 zero_point = 0; + int32_t zero_point = 0; double scale = 0.0; }; @@ -140,20 +145,20 @@ class RuntimeShape { if (dimensions_count > kMaxSmallSize) { #ifdef TF_LITE_STATIC_MEMORY TFLITE_CHECK(false && "No shape resizing supported on this platform"); -#else // TF_LITE_STATIC_MEMORY - dims_pointer_ = new int32[dimensions_count]; +#else // TF_LITE_STATIC_MEMORY + dims_pointer_ = new int32_t[dimensions_count]; #endif // TF_LITE_STATIC_MEMORY } } - RuntimeShape(int shape_size, int32 value) : size_(0) { + RuntimeShape(int shape_size, int32_t value) : size_(0) { Resize(shape_size); for (int i = 0; i < shape_size; ++i) { SetDim(i, value); } } - RuntimeShape(int dimensions_count, const int32* dims_data) : size_(0) { + RuntimeShape(int dimensions_count, const int32_t* dims_data) : size_(0) { ReplaceWith(dimensions_count, dims_data); } @@ -165,33 +170,34 @@ class RuntimeShape { // rolls out. RuntimeShape(RuntimeShape const& other) : size_(other.DimensionsCount()) { if (size_ > kMaxSmallSize) { - dims_pointer_ = new int32[size_]; + dims_pointer_ = new int32_t[size_]; } - std::memcpy(DimsData(), other.DimsData(), sizeof(int32) * size_); + std::memcpy(DimsData(), other.DimsData(), sizeof(int32_t) * size_); } bool operator==(const RuntimeShape& comp) const { return this->size_ == comp.size_ && - std::memcmp(DimsData(), comp.DimsData(), size_ * sizeof(int32)) == 0; + std::memcmp(DimsData(), comp.DimsData(), size_ * sizeof(int32_t)) == + 0; } ~RuntimeShape() { if (size_ > kMaxSmallSize) { #ifdef TF_LITE_STATIC_MEMORY TFLITE_CHECK(false && "No shape resizing supported on this platform"); -#else // TF_LITE_STATIC_MEMORY +#else // TF_LITE_STATIC_MEMORY delete[] dims_pointer_; #endif // TF_LITE_STATIC_MEMORY } } - inline int32 DimensionsCount() const { return size_; } - inline int32 Dims(int i) const { + inline int32_t DimensionsCount() const { return size_; } + inline int32_t Dims(int i) const { TFLITE_DCHECK_GE(i, 0); TFLITE_DCHECK_LT(i, size_); return size_ > kMaxSmallSize ? dims_pointer_[i] : dims_[i]; } - inline void SetDim(int i, int32 val) { + inline void SetDim(int i, int32_t val) { TFLITE_DCHECK_GE(i, 0); TFLITE_DCHECK_LT(i, size_); if (size_ > kMaxSmallSize) { @@ -201,20 +207,20 @@ class RuntimeShape { } } - inline int32* DimsData() { + inline int32_t* DimsData() { return size_ > kMaxSmallSize ? dims_pointer_ : dims_; } - inline const int32* DimsData() const { + inline const int32_t* DimsData() const { return size_ > kMaxSmallSize ? dims_pointer_ : dims_; } // The caller must ensure that the shape is no bigger than 5-D. - inline const int32* DimsDataUpTo5D() const { return dims_; } + inline const int32_t* DimsDataUpTo5D() const { return dims_; } inline void Resize(int dimensions_count) { if (size_ > kMaxSmallSize) { #ifdef TF_LITE_STATIC_MEMORY TFLITE_CHECK(false && "No shape resizing supported on this platform"); -#else // TF_LITE_STATIC_MEMORY +#else // TF_LITE_STATIC_MEMORY delete[] dims_pointer_; #endif // TF_LITE_STATIC_MEMORY } @@ -222,16 +228,16 @@ class RuntimeShape { if (dimensions_count > kMaxSmallSize) { #ifdef TF_LITE_STATIC_MEMORY TFLITE_CHECK(false && "No shape resizing supported on this platform"); -#else // TF_LITE_STATIC_MEMORY - dims_pointer_ = new int32[dimensions_count]; +#else // TF_LITE_STATIC_MEMORY + dims_pointer_ = new int32_t[dimensions_count]; #endif // TF_LITE_STATIC_MEMORY } } - inline void ReplaceWith(int dimensions_count, const int32* dims_data) { + inline void ReplaceWith(int dimensions_count, const int32_t* dims_data) { Resize(dimensions_count); - int32* dst_dims = DimsData(); - std::memcpy(dst_dims, dims_data, dimensions_count * sizeof(int32)); + int32_t* dst_dims = DimsData(); + std::memcpy(dst_dims, dims_data, dimensions_count * sizeof(int32_t)); } template @@ -239,7 +245,7 @@ class RuntimeShape { const int dimensions_count = std::distance(src_iterable.begin(), src_iterable.end()); Resize(dimensions_count); - int32* data = DimsData(); + int32_t* data = DimsData(); for (auto it : src_iterable) { *data = it; ++data; @@ -288,13 +294,13 @@ class RuntimeShape { SetDim(i, pad_value); } std::memcpy(DimsData() + size_increase, shape.DimsData(), - sizeof(int32) * shape.DimensionsCount()); + sizeof(int32_t) * shape.DimensionsCount()); } - int32 size_; + int32_t size_; union { - int32 dims_[kMaxSmallSize]; - int32* dims_pointer_; + int32_t dims_[kMaxSmallSize]; + int32_t* dims_pointer_; }; }; @@ -432,7 +438,7 @@ int MatchingArraySize(const ArrayType1& array1, int index1, inline int MatchingDim(const RuntimeShape& shape1, int index1, const RuntimeShape& shape2, int index2) { TFLITE_DCHECK_EQ(shape1.Dims(index1), shape2.Dims(index2)); - return shape1.Dims(index1); + return std::min(shape1.Dims(index1), shape2.Dims(index2)); } template @@ -713,7 +719,7 @@ void ComputeStrides(Dims* dims) { } } -enum class BroadcastableOpCategory : uint8 { +enum class BroadcastableOpCategory : uint8_t { kNone, kNonBroadcast, // Matching input shapes. kFirstInputBroadcastsFast, // Fivefold nested loops. @@ -729,21 +735,21 @@ static_assert(sizeof(MinMax) == 8, ""); struct ActivationParams { FusedActivationFunctionType activation_type; - // uint8, etc, activation params. - int32 quantized_activation_min; - int32 quantized_activation_max; + // uint8_t, etc, activation params. + int32_t quantized_activation_min; + int32_t quantized_activation_max; }; struct ReluParams : public ActivationParams { - int32 input_offset; - int32 output_offset; - int32 output_multiplier; - int32 output_shift; + int32_t input_offset; + int32_t output_offset; + int32_t output_multiplier; + int output_shift; }; // Styles of resizing op usages. For example, kImageStyle can be used with a Pad // op for pattern-specific optimization. -enum class ResizingCategory : uint8 { +enum class ResizingCategory : uint8_t { kNone, kImageStyle, // 4D, operating on inner dimensions, say {0, a, b, 0}. kGenericResize, @@ -753,24 +759,29 @@ enum class ResizingCategory : uint8 { struct ArithmeticParams { // Shape dependent / common to data / op types. BroadcastableOpCategory broadcast_category; - // uint8 inference params. - int32 input1_offset; - int32 input2_offset; - int32 output_offset; - int32 output_multiplier; + // uint8_t inference params. + int32_t input1_offset; + int32_t input2_offset; + int32_t output_offset; + int32_t output_multiplier; int output_shift; - // Add / Sub, not Mul, uint8 inference params. + // Add / Sub, not Mul, uint8_t inference params. int left_shift; - int32 input1_multiplier; + int32_t input1_multiplier; int input1_shift; - int32 input2_multiplier; + int32_t input2_multiplier; int input2_shift; - // uint8, etc, activation params. - int32 quantized_activation_min; - int32 quantized_activation_max; + + // TODO(b/158622529): Union the following activation params. + // uint8_t, etc, activation params. + int32_t quantized_activation_min; + int32_t quantized_activation_max; // float activation params. float float_activation_min; float float_activation_max; + // int64_t activation params. + int64_t int64_activation_min; + int64_t int64_activation_max; // Processed output dimensions. // Let input "a" be the one that broadcasts in the faster-changing dimension. @@ -785,22 +796,22 @@ struct ArithmeticParams { }; struct ConcatenationParams { - int8 axis; - const int32* input_zeropoint; + int8_t axis; + const int32_t* input_zeropoint; const float* input_scale; - uint16 inputs_count; - int32 output_zeropoint; + uint16_t inputs_count; + int32_t output_zeropoint; float output_scale; }; struct ComparisonParams { - // uint8 inference params. + // uint8_t inference params. int left_shift; - int32 input1_offset; - int32 input1_multiplier; + int32_t input1_offset; + int32_t input1_multiplier; int input1_shift; - int32 input2_offset; - int32 input2_multiplier; + int32_t input2_offset; + int32_t input2_multiplier; int input2_shift; // Shape dependent / common to inference types. bool is_broadcast; @@ -810,81 +821,81 @@ struct ConvParams { PaddingType padding_type; PaddingValues padding_values; // TODO(starka): This was just "stride", so check that width+height is OK. - int16 stride_width; - int16 stride_height; - int16 dilation_width_factor; - int16 dilation_height_factor; - // uint8 inference params. + int16_t stride_width; + int16_t stride_height; + int16_t dilation_width_factor; + int16_t dilation_height_factor; + // uint8_t inference params. // TODO(b/65838351): Use smaller types if appropriate. - int32 input_offset; - int32 weights_offset; - int32 output_offset; - int32 output_multiplier; + int32_t input_offset; + int32_t weights_offset; + int32_t output_offset; + int32_t output_multiplier; int output_shift; - // uint8, etc, activation params. - int32 quantized_activation_min; - int32 quantized_activation_max; + // uint8_t, etc, activation params. + int32_t quantized_activation_min; + int32_t quantized_activation_max; // float activation params. float float_activation_min; float float_activation_max; }; struct DepthToSpaceParams { - int32 block_size; + int32_t block_size; }; struct DepthwiseParams { PaddingType padding_type; PaddingValues padding_values; - int16 stride_width; - int16 stride_height; - int16 dilation_width_factor; - int16 dilation_height_factor; - int16 depth_multiplier; - // uint8 inference params. + int16_t stride_width; + int16_t stride_height; + int16_t dilation_width_factor; + int16_t dilation_height_factor; + int16_t depth_multiplier; + // uint8_t inference params. // TODO(b/65838351): Use smaller types if appropriate. - int32 input_offset; - int32 weights_offset; - int32 output_offset; - int32 output_multiplier; + int32_t input_offset; + int32_t weights_offset; + int32_t output_offset; + int32_t output_multiplier; int output_shift; - // uint8, etc, activation params. - int32 quantized_activation_min; - int32 quantized_activation_max; + // uint8_t, etc, activation params. + int32_t quantized_activation_min; + int32_t quantized_activation_max; // float activation params. float float_activation_min; float float_activation_max; - const int32* output_multiplier_per_channel; - const int32* output_shift_per_channel; + const int32_t* output_multiplier_per_channel; + const int32_t* output_shift_per_channel; }; struct DequantizationParams { double scale; - int32 zero_point; + int32_t zero_point; }; struct PerChannelDequantizationParams { const float* scale; - const int32* zero_point; - int32 quantized_dimension; + const int32_t* zero_point; + int32_t quantized_dimension; }; struct FakeQuantParams { MinMax minmax; - int32 num_bits; + int32_t num_bits; }; struct FullyConnectedParams { - // uint8 inference params. + // uint8_t inference params. // TODO(b/65838351): Use smaller types if appropriate. - int32 input_offset; - int32 weights_offset; - int32 output_offset; - int32 output_multiplier; + int32_t input_offset; + int32_t weights_offset; + int32_t output_offset; + int32_t output_multiplier; int output_shift; - // uint8, etc, activation params. - int32 quantized_activation_min; - int32 quantized_activation_max; + // uint8_t, etc, activation params. + int32_t quantized_activation_min; + int32_t quantized_activation_max; // float activation params. float float_activation_min; float float_activation_max; @@ -895,16 +906,16 @@ struct FullyConnectedParams { }; struct GatherParams { - int16 axis; + int16_t axis; }; struct L2NormalizationParams { - // uint8 inference params. - int32 input_zero_point; + // uint8_t inference params. + int32_t input_zero_point; }; struct LocalResponseNormalizationParams { - int32 range; + int32_t range; double bias; double alpha; double beta; @@ -932,48 +943,50 @@ struct HardSwishParams { }; struct LogisticParams { - // uint8 inference params. - int32 input_zero_point; - int32 input_range_radius; - int32 input_multiplier; + // uint8_t inference params. + int32_t input_zero_point; + int32_t input_range_radius; + int32_t input_multiplier; int input_left_shift; }; struct LstmCellParams { - int32 weights_zero_point; - int32 accum_multiplier; + int32_t weights_zero_point; + int32_t accum_multiplier; int accum_shift; int state_integer_bits; }; struct MeanParams { - int8 axis_count; - int16 axis[4]; + int8_t axis_count; + int16_t axis[4]; }; struct PackParams { - int8 axis; - const int32* input_zeropoint; + int8_t axis; + const int32_t* input_zeropoint; const float* input_scale; - uint16 inputs_count; - int32 output_zeropoint; + uint16_t inputs_count; + int32_t output_zeropoint; float output_scale; }; struct PadParams { - int8 left_padding_count; - int32 left_padding[4]; - int8 right_padding_count; - int32 right_padding[4]; + int8_t left_padding_count; + int32_t left_padding[4]; + int8_t right_padding_count; + int32_t right_padding[4]; ResizingCategory resizing_category; }; struct PreluParams { - int32 input_offset; - int32 alpha_offset; - int32 output_offset; - int32 output_multiplier; - int output_shift; + int32_t input_offset; + int32_t alpha_offset; + int32_t output_offset; + int32_t output_multiplier_1; + int output_shift_1; + int32_t output_multiplier_2; + int output_shift_2; }; struct PoolParams { @@ -984,17 +997,17 @@ struct PoolParams { int stride_width; int filter_height; int filter_width; - // uint8, etc, activation params. - int32 quantized_activation_min; - int32 quantized_activation_max; + // uint8_t, etc, activation params. + int32_t quantized_activation_min; + int32_t quantized_activation_max; // float activation params. float float_activation_min; float float_activation_max; }; struct ReshapeParams { - int8 shape_count; - int32 shape[4]; + int8_t shape_count; + int32_t shape[4]; }; struct ResizeBilinearParams { @@ -1011,91 +1024,95 @@ struct ResizeNearestNeighborParams { }; struct SliceParams { - int8 begin_count; - int32 begin[4]; - int8 size_count; - int32 size[4]; + int8_t begin_count; + int32_t begin[4]; + int8_t size_count; + int32_t size[4]; }; struct SoftmaxParams { // beta is not really used (not a Tensorflow parameter) and not implemented // for LogSoftmax. double beta; - // uint8 inference params. Used even when beta defaults to 1.0. - int32 input_multiplier; - int32 input_left_shift; + // uint8_t inference params. Used even when beta defaults to 1.0. + int32_t input_multiplier; + int32_t input_left_shift; // Reverse scaling is only used by LogSoftmax. - int32 reverse_scaling_divisor; - int32 reverse_scaling_right_shift; + int32_t reverse_scaling_divisor; + int32_t reverse_scaling_right_shift; int diff_min; int32_t zero_point; float scale; float* table; + // int16 LUT for exp(x), where x uniform distributed between [-10.0 , 0.0] int16_t* exp_lut; + // int16 LUT for 1 / (1 + x), where x uniform distributed between [0.0 , 1.0] int16_t* one_over_one_plus_x_lut; + uint8_t* uint8_table1; + uint8_t* uint8_table2; }; struct SpaceToBatchParams { - // "Zero" padding for uint8 means padding with the output offset. - int32 output_offset; + // "Zero" padding for uint8_t means padding with the output offset. + int32_t output_offset; }; struct SpaceToDepthParams { - int32 block_size; + int32_t block_size; }; struct SplitParams { // Graphs that split into, say, 2000 nodes are encountered. The indices in - // OperatorEdges are of type uint16. - uint16 num_split; - int16 axis; + // OperatorEdges are of type uint16_t. + uint16_t num_split; + int16_t axis; }; struct SqueezeParams { - int8 squeeze_dims_count; - int32 squeeze_dims[4]; + int8_t squeeze_dims_count; + int32_t squeeze_dims[4]; }; struct StridedSliceParams { - int8 start_indices_count; - int32 start_indices[5]; - int8 stop_indices_count; - int32 stop_indices[5]; - int8 strides_count; - int32 strides[5]; + int8_t start_indices_count; + int32_t start_indices[5]; + int8_t stop_indices_count; + int32_t stop_indices[5]; + int8_t strides_count; + int32_t strides[5]; - int16 begin_mask; - int16 ellipsis_mask; - int16 end_mask; - int16 new_axis_mask; - int16 shrink_axis_mask; + int16_t begin_mask; + int16_t ellipsis_mask; + int16_t end_mask; + int16_t new_axis_mask; + int16_t shrink_axis_mask; }; struct TanhParams { - int32 input_zero_point; - int32 input_range_radius; - int32 input_multiplier; + int32_t input_zero_point; + int32_t input_range_radius; + int32_t input_multiplier; int input_left_shift; }; struct TransposeParams { - int8 perm_count; - int32 perm[5]; + int8_t perm_count; + int32_t perm[5]; }; struct UnpackParams { - uint16 num_split; - int16 axis; + uint16_t num_split; + int16_t axis; }; struct LeakyReluParams { float alpha; - int32 input_offset; - int32 output_offset; - int32 output_multiplier_alpha; - int32 output_shift_alpha; - int32 output_multiplier_identity; - int32 output_shift_identity; + int32_t input_offset; + int32_t output_offset; + int32_t output_multiplier_alpha; + int32_t output_shift_alpha; + int32_t output_multiplier_identity; + int32_t output_shift_identity; }; template @@ -1105,13 +1122,19 @@ inline void SetActivationParams(float min, float max, P* params) { } template -inline void SetActivationParams(int32 min, int32 max, P* params) { +inline void SetActivationParams(int32_t min, int32_t max, P* params) { params->quantized_activation_min = min; params->quantized_activation_max = max; } template -inline void GetActivationParams(const P& params, int32* min, int32* max) { +inline void SetActivationParams(int64_t min, int64_t max, P* params) { + params->int64_activation_min = min; + params->int64_activation_max = max; +} + +template +inline void GetActivationParams(const P& params, int32_t* min, int32_t* max) { *min = params.quantized_activation_min; *max = params.quantized_activation_max; } @@ -1122,6 +1145,11 @@ inline void GetActivationParams(const P& params, float* min, float* max) { *max = params.float_activation_max; } +template +inline void GetActivationParams(const P& params, int64_t* min, int64_t* max) { + *min = params.int64_activation_min; + *max = params.int64_activation_max; +} } // namespace tflite #endif // TENSORFLOW_LITE_KERNELS_INTERNAL_TYPES_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/kernel_util.cc b/code/lib/tfmicro/tensorflow/lite/kernels/kernel_util.cc index b30747ea..f986655f 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/kernel_util.cc +++ b/code/lib/tfmicro/tensorflow/lite/kernels/kernel_util.cc @@ -14,15 +14,176 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/kernels/kernel_util.h" +#include +#include + #include -#include +#include +#include #include +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/cppmath.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" namespace tflite { +namespace { + +// Assumes tensor_index is a valid index (in bounds) +inline TfLiteTensor* GetTensorAtIndex(const TfLiteContext* context, + int tensor_index) { + if (context->tensors != nullptr) { + return &context->tensors[tensor_index]; + } else { + return context->GetTensor(context, tensor_index); + } +} + +// Validate in a single place to reduce binary size +inline TfLiteStatus ValidateTensorIndexingSafe(const TfLiteContext* context, + int index, int max_size, + const int* tensor_indices, + int* tensor_index) { + if (index < 0 || index >= max_size) { + TF_LITE_KERNEL_LOG(const_cast(context), + "Invalid tensor index %d (not in [0, %d))\n", index, + max_size); + return kTfLiteError; + } + if (tensor_indices[index] == kTfLiteOptionalTensor) { + TF_LITE_KERNEL_LOG(const_cast(context), + "Tensor at index %d was optional but was expected\n", + index); + return kTfLiteError; + } + + *tensor_index = tensor_indices[index]; + return kTfLiteOk; +} + +// Same as above but returns -1 for invalid inputs instead of status + logging +// error. +inline int ValidateTensorIndexing(const TfLiteContext* context, int index, + int max_size, const int* tensor_indices) { + if (index >= 0 && index < max_size) { + const int tensor_index = tensor_indices[index]; + if (tensor_index != kTfLiteOptionalTensor) { + return tensor_index; + } + } + return -1; +} + +inline TfLiteTensor* GetMutableInput(const TfLiteContext* context, + const TfLiteNode* node, int index) { + const int tensor_index = ValidateTensorIndexing( + context, index, node->inputs->size, node->inputs->data); + if (tensor_index < 0) { + return nullptr; + } + return GetTensorAtIndex(context, tensor_index); +} + +inline TfLiteStatus GetMutableInputSafe(const TfLiteContext* context, + const TfLiteNode* node, int index, + const TfLiteTensor** tensor) { + int tensor_index; + TF_LITE_ENSURE_OK( + context, ValidateTensorIndexingSafe(context, index, node->inputs->size, + node->inputs->data, &tensor_index)); + *tensor = GetTensorAtIndex(context, tensor_index); + return kTfLiteOk; +} + +} // anonymous namespace. + +const TfLiteTensor* GetInput(const TfLiteContext* context, + const TfLiteNode* node, int index) { + return GetMutableInput(context, node, index); +} + +TfLiteStatus GetInputSafe(const TfLiteContext* context, const TfLiteNode* node, + int index, const TfLiteTensor** tensor) { + return GetMutableInputSafe(context, node, index, tensor); +} + +TfLiteTensor* GetVariableInput(TfLiteContext* context, const TfLiteNode* node, + int index) { + TfLiteTensor* tensor = GetMutableInput(context, node, index); + return tensor->is_variable ? tensor : nullptr; +} + +TfLiteTensor* GetOutput(TfLiteContext* context, const TfLiteNode* node, + int index) { + const int tensor_index = ValidateTensorIndexing( + context, index, node->outputs->size, node->outputs->data); + if (tensor_index < 0) { + return nullptr; + } + return GetTensorAtIndex(context, tensor_index); +} + +TfLiteStatus GetOutputSafe(const TfLiteContext* context, const TfLiteNode* node, + int index, TfLiteTensor** tensor) { + int tensor_index; + TF_LITE_ENSURE_OK( + context, ValidateTensorIndexingSafe(context, index, node->outputs->size, + node->outputs->data, &tensor_index)); + *tensor = GetTensorAtIndex(context, tensor_index); + return kTfLiteOk; +} + +const TfLiteTensor* GetOptionalInputTensor(const TfLiteContext* context, + const TfLiteNode* node, int index) { + return GetInput(context, node, index); +} + +#ifndef TF_LITE_STATIC_MEMORY +TfLiteTensor* GetTemporary(TfLiteContext* context, const TfLiteNode* node, + int index) { + const int tensor_index = ValidateTensorIndexing( + context, index, node->temporaries->size, node->temporaries->data); + if (tensor_index < 0) { + return nullptr; + } + return GetTensorAtIndex(context, tensor_index); +} + +TfLiteStatus GetTemporarySafe(const TfLiteContext* context, + const TfLiteNode* node, int index, + TfLiteTensor** tensor) { + int tensor_index; + TF_LITE_ENSURE_OK(context, ValidateTensorIndexingSafe( + context, index, node->temporaries->size, + node->temporaries->data, &tensor_index)); + *tensor = GetTensorAtIndex(context, tensor_index); + return kTfLiteOk; +} + +const TfLiteTensor* GetIntermediates(TfLiteContext* context, + const TfLiteNode* node, int index) { + const int tensor_index = ValidateTensorIndexing( + context, index, node->intermediates->size, node->intermediates->data); + if (tensor_index < 0) { + return nullptr; + } + return GetTensorAtIndex(context, tensor_index); +} + +TfLiteStatus GetIntermediatesSafe(const TfLiteContext* context, + const TfLiteNode* node, int index, + TfLiteTensor** tensor) { + int tensor_index; + TF_LITE_ENSURE_OK(context, ValidateTensorIndexingSafe( + context, index, node->intermediates->size, + node->intermediates->data, &tensor_index)); + *tensor = GetTensorAtIndex(context, tensor_index); + return kTfLiteOk; +} +#endif // TF_LITE_STATIC_MEMORY + // Per-axis TfLiteStatus PopulateConvolutionQuantizationParams( TfLiteContext* context, const TfLiteTensor* input, @@ -126,11 +287,27 @@ TfLiteStatus GetQuantizedConvolutionMultipler(TfLiteContext* context, // pipeline. if (bias) { const double bias_scale = static_cast(bias->params.scale); - // Here we're making sure the input_product_scale & bias_scale the same. - // Normally this should be guaranteed by the training pipeline, we are - // setting the threshold to be 2e-6 to allow some numeric stability - // difference. - TF_LITE_ENSURE(context, std::abs(input_product_scale - bias_scale) <= 2e-6); + // Here we're making sure the input_product_scale & bias_scale are about the + // same. Since we have: + // (output - output_zp) * output_scale = + // input_product_scale * input_product + bias * bias_scale ---- (0) + // + // (0) equals: + // (input_product + bias) * input_product_scale ----- (1) + // + + // bias * (bias_scale - input_product_scale) ------ (2) + // + // For the real kernel computation, we're doing (1), so we really need to + // make sure (2) has minimum impact on the output, so: + // bias * (bias_scale - input_product_scale) / output_scale should be + // a small number for an integer. + // Since normally bias should be within a small range. + // We should expect (bias_scale - input_product_scale) / output_scale to + // be a small number like 0.02. + const double scale_diff = std::abs(input_product_scale - bias_scale); + const double output_scale = static_cast(output->params.scale); + + TF_LITE_ENSURE(context, scale_diff / output_scale <= 0.02); } return GetQuantizedConvolutionMultipler(context, input, filter, output, multiplier); @@ -167,7 +344,7 @@ void CalculateActivationRangeQuantizedImpl(TfLiteFusedActivation activation, } else if (activation == kTfLiteActRelu6) { *act_min = std::max(qmin, quantize(0.0)); *act_max = std::min(qmax, quantize(6.0)); - } else if (activation == kTfLiteActRelu1) { + } else if (activation == kTfLiteActReluN1To1) { *act_min = std::max(qmin, quantize(-1.0)); *act_max = std::min(qmax, quantize(1.0)); } else { @@ -258,4 +435,44 @@ TfLiteStatus CalculateShapeForBroadcast(TfLiteContext* context, } #endif // TF_LITE_STATIC_MEMORY +// Size of string is not constant, return 0 in such case. +int TfLiteTypeGetSize(TfLiteType type) { + switch (type) { + case kTfLiteUInt8: + TF_LITE_ASSERT_EQ(sizeof(uint8_t), 1); + return 1; + case kTfLiteInt8: + TF_LITE_ASSERT_EQ(sizeof(int8_t), 1); + return 1; + case kTfLiteBool: + return sizeof(bool); + case kTfLiteInt16: + TF_LITE_ASSERT_EQ(sizeof(int16_t), 2); + return 2; + case kTfLiteFloat16: + TF_LITE_ASSERT_EQ(sizeof(int16_t), 2); + return 2; + case kTfLiteFloat32: + TF_LITE_ASSERT_EQ(sizeof(float), 4); + return 4; + case kTfLiteInt32: + TF_LITE_ASSERT_EQ(sizeof(int32_t), 4); + return 4; + case kTfLiteInt64: + TF_LITE_ASSERT_EQ(sizeof(int64_t), 8); + return 8; + case kTfLiteFloat64: + TF_LITE_ASSERT_EQ(sizeof(double), 8); + return 8; + case kTfLiteComplex64: + TF_LITE_ASSERT_EQ(sizeof(std::complex), 8); + return 8; + case kTfLiteComplex128: + TF_LITE_ASSERT_EQ(sizeof(std::complex), 16); + return 16; + default: + return 0; + } +} + } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/kernel_util.h b/code/lib/tfmicro/tensorflow/lite/kernels/kernel_util.h index ad068ddd..7a1aa165 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/kernel_util.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/kernel_util.h @@ -15,52 +15,148 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_KERNEL_UTIL_H_ #define TENSORFLOW_LITE_KERNELS_KERNEL_UTIL_H_ -#include +#include + #include -#include "flatbuffers/flatbuffers.h" #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" namespace tflite { +// A fair number of functions in this header have historically been inline. +// It is ok to change functions to not be inline if the latency with +// benchmark_model for MobileNet + MobileBERT is unaffected. If such a change is +// made, move the newly non-inlined function declarations to the top of this +// header file. + +// Note: You must check if result is not null: +// +// TfLiteTensor* my_tensor = GetInput(context, node, kMyTensorIdx); +// TF_LITE_ENSURE(context, my_tensor != nullptr); +// +// This is because the index might point to the optional tensor constant +// (kTfLiteOptionalTensor) in which case there is no tensor to return. +const TfLiteTensor* GetInput(const TfLiteContext* context, + const TfLiteNode* node, int index); + +// Same as `GetInput` but returns boolean and uses output argument for tensor. +// +// TfLiteTensor* my_tensor; +// TF_LITE_ENSURE_OK(context, +// GetInputSafe(context, node, kMyTensorIdx, &my_tensor)); +// // can use my_tensor directly from here onwards, it is not nullptr +// +// Should be used in cases where the binary size is too large. +TfLiteStatus GetInputSafe(const TfLiteContext* context, const TfLiteNode* node, + int index, const TfLiteTensor** tensor); + +// Note: You must check if result is not null: +// +// TfLiteTensor* my_tensor = GetVariableInput(context, node, kMyTensorIdx); +// TF_LITE_ENSURE(context, my_tensor != nullptr); +// +// This is because the index might point to the optional tensor constant +// (kTfLiteOptionalTensor) in which case there is no tensor to return. +TfLiteTensor* GetVariableInput(TfLiteContext* context, const TfLiteNode* node, + int index); + +// Note: You must check if result is not null: +// +// TfLiteTensor* my_tensor = GetOutput(context, node, kMyTensorIdx); +// TF_LITE_ENSURE(context, my_tensor != nullptr); +// +// This is because the index might point to the optional tensor constant +// (kTfLiteOptionalTensor) in which case there is no tensor to return. +TfLiteTensor* GetOutput(TfLiteContext* context, const TfLiteNode* node, + int index); + +// Same as `GetOutput` but returns boolean and uses output argument for tensor. +// +// TfLiteTensor* my_tensor; +// TF_LITE_ENSURE_OK(context, +// GetOutputSafe(context, node, kMyTensorIdx, &my_tensor)); +// // can use my_tensor directly from here onwards, it is not nullptr +// +// Should be used in cases where the binary size is too large. +TfLiteStatus GetOutputSafe(const TfLiteContext* context, const TfLiteNode* node, + int index, TfLiteTensor** tensor); + +// Note: You must check if result is not null: +// +// TfLiteTensor* my_tensor = GetOptionalInputTensor(context, node, kIdx); +// TF_LITE_ENSURE(context, my_tensor != nullptr); +// +// This is because the index might point to the optional tensor constant +// (kTfLiteOptionalTensor) in which case there is no tensor to return. +// +// Deprecated. GetInput has the same functionality. +const TfLiteTensor* GetOptionalInputTensor(const TfLiteContext* context, + const TfLiteNode* node, int index); + +#ifndef TF_LITE_STATIC_MEMORY +// Note: You must check if result is not null: +// +// TfLiteTensor* my_tensor = GetTemporary(context, node, kMyTensorIdx); +// TF_LITE_ENSURE(context, my_tensor != nullptr); +// +// This is because the index might point to the optional tensor constant +// (kTfLiteOptionalTensor) in which case there is no tensor to return. +TfLiteTensor* GetTemporary(TfLiteContext* context, const TfLiteNode* node, + int index); + +// Same as `GetTemporary` but returns boolean and uses output argument for +// tensor. +// +// TfLiteTensor* my_tensor; +// TF_LITE_ENSURE_OK(context, +// GetTemporarySafe(context, node, kMyTensorIdx, +// &my_tensor)); +// // can use my_tensor directly from here onwards, it is not nullptr +// +// Should be used in cases where the binary size is too large. +TfLiteStatus GetTemporarySafe(const TfLiteContext* context, + const TfLiteNode* node, int index, + TfLiteTensor** tensor); + +// Note: You must check if result is not null: +// +// TfLiteTensor* my_tensor = GetIntermediates(context, node, kMyTensorIdx); +// TF_LITE_ENSURE(context, my_tensor != nullptr); +// +// This is because the index might point to the optional tensor constant +// (kTfLiteOptionalTensor) in which case there is no tensor to return. +const TfLiteTensor* GetIntermediates(TfLiteContext* context, + const TfLiteNode* node, int index); + +// Same as `GetIntermediates` but returns boolean and uses output argument for +// tensor. +// +// TfLiteTensor* my_tensor; +// TF_LITE_ENSURE_OK(context, +// GetIntermediatesSafe(context, node, kMyTensorIdx, +// &my_tensor)); +// // can use my_tensor directly from here onwards, it is not nullptr +// +// Should be used in cases where the binary size is too large. +TfLiteStatus GetIntermediatesSafe(const TfLiteContext* context, + const TfLiteNode* node, int index, + TfLiteTensor** tensor); +#endif // TF_LITE_STATIC_MEMORY + inline int NumDimensions(const TfLiteTensor* t) { return t->dims->size; } inline int SizeOfDimension(const TfLiteTensor* t, int dim) { return t->dims->data[dim]; } -inline const TfLiteTensor* GetInput(TfLiteContext* context, - const TfLiteNode* node, int index) { - return &context - ->tensors[flatbuffers::EndianScalar(node->inputs->data[index])]; -} -// Note: You must check if result is not null: -// TfLiteTensor* my_tensor = GetVariableInput(context, node, kMyTensorIdx); -// TF_LITE_ENSURE(context, my_tensor != nullptr); -inline TfLiteTensor* GetVariableInput(TfLiteContext* context, - const TfLiteNode* node, int index) { - TfLiteTensor* tensor = - &context->tensors[flatbuffers::EndianScalar(node->inputs->data[index])]; - return (tensor->is_variable) ? tensor : nullptr; -} -inline TfLiteTensor* GetOutput(TfLiteContext* context, const TfLiteNode* node, - int index) { - return &context - ->tensors[flatbuffers::EndianScalar(node->outputs->data[index])]; -} -inline TfLiteTensor* GetTemporary(TfLiteContext* context, - const TfLiteNode* node, int index) { - return &context->tensors[flatbuffers::EndianScalar( - node->temporaries->data[index])]; -} -inline const TfLiteTensor* GetIntermediates(TfLiteContext* context, - const TfLiteNode* node, int index) { - return &context->tensors[node->intermediates->data[index]]; -} + inline int NumInputs(const TfLiteNode* node) { return node->inputs->size; } inline int NumOutputs(const TfLiteNode* node) { return node->outputs->size; } + +#ifndef TF_LITE_STATIC_MEMORY inline int NumIntermediates(const TfLiteNode* node) { return node->intermediates->size; } +#endif // TF_LITE_STATIC_MEMORY inline int64_t NumElements(const TfLiteIntArray* dims) { int64_t count = 1; @@ -74,19 +170,11 @@ inline int64_t NumElements(const TfLiteTensor* t) { return NumElements(t->dims); } -inline const TfLiteTensor* GetOptionalInputTensor(TfLiteContext* context, - const TfLiteNode* node, - int index) { - const bool use_tensor = index < node->inputs->size && - node->inputs->data[index] != kTfLiteOptionalTensor; - if (use_tensor) { - return &context - ->tensors[flatbuffers::EndianScalar(node->inputs->data[index])]; - } - return nullptr; -} - // Determines whether tensor is constant. +// TODO(b/138199592): Introduce new query which checks for constant OR +// persistent-read-only, which would be useful for most tensor kernels that +// are potentially dynamic based on the input tensor value availability at the +// time of prepare. inline bool IsConstantTensor(const TfLiteTensor* tensor) { return tensor->allocation_type == kTfLiteMmapRo; } @@ -105,6 +193,14 @@ inline void SetTensorToDynamic(TfLiteTensor* tensor) { } } +// Sets tensor to persistent and read-only. +inline void SetTensorToPersistentRo(TfLiteTensor* tensor) { + if (tensor->allocation_type != kTfLitePersistentRo) { + tensor->allocation_type = kTfLitePersistentRo; + tensor->data.raw = nullptr; + } +} + // Determines whether it is a hybrid op - one that has float inputs and // quantized weights. inline bool IsHybridOp(const TfLiteTensor* input, const TfLiteTensor* weight) { @@ -162,7 +258,7 @@ void CalculateActivationRange(TfLiteFusedActivation activation, } else if (activation == kTfLiteActRelu6) { *activation_min = 0; *activation_max = 6; - } else if (activation == kTfLiteActRelu1) { + } else if (activation == kTfLiteActReluN1To1) { *activation_min = -1; *activation_max = 1; } else { @@ -188,6 +284,10 @@ TfLiteStatus CalculateShapeForBroadcast(TfLiteContext* context, const TfLiteTensor* input2, const TfLiteTensor* input3, TfLiteIntArray** output_shape); + +// Return the size of given type in bytes. Return 0 in in case of string. +int TfLiteTypeGetSize(TfLiteType type); + } // namespace tflite #endif // TENSORFLOW_LITE_KERNELS_KERNEL_UTIL_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/kernels/op_macros.h b/code/lib/tfmicro/tensorflow/lite/kernels/op_macros.h index 33d033b1..5786756f 100644 --- a/code/lib/tfmicro/tensorflow/lite/kernels/op_macros.h +++ b/code/lib/tfmicro/tensorflow/lite/kernels/op_macros.h @@ -19,7 +19,7 @@ limitations under the License. // non-portable function. #ifdef TF_LITE_MCU_DEBUG_LOG -#include "tensorflow/lite/micro/micro_error_reporter.h" +#include "tensorflow/lite/micro/debug_log.h" #define DEBUG_LOG(x) \ do { \ @@ -36,7 +36,6 @@ inline void InfiniteLoop() { #else // TF_LITE_MCU_DEBUG_LOG -#include #include #include @@ -45,6 +44,15 @@ inline void InfiniteLoop() { fprintf(stderr, "%s", (x)); \ } while (0) +// Report Error for unsupported type by op 'op_name' and returns kTfLiteError. +#define TF_LITE_UNSUPPORTED_TYPE(context, type, op_name) \ + do { \ + TF_LITE_KERNEL_LOG((context), "%s:%d Type %s is unsupported by op %s.", \ + __FILE__, __LINE__, TfLiteTypeGetName(type), \ + (op_name)); \ + return kTfLiteError; \ + } while (0) + #define TFLITE_ABORT abort() #endif // TF_LITE_MCU_DEBUG_LOG diff --git a/code/lib/tfmicro/tensorflow/lite/micro/all_ops_resolver.cc b/code/lib/tfmicro/tensorflow/lite/micro/all_ops_resolver.cc new file mode 100644 index 00000000..0a2a0c0f --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/all_ops_resolver.cc @@ -0,0 +1,94 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/all_ops_resolver.h" + +#include "tensorflow/lite/micro/kernels/micro_ops.h" + +namespace tflite { +namespace ops { +namespace micro { +namespace custom { +TfLiteRegistration* Register_ETHOSU(); +const char* GetString_ETHOSU(); +} // namespace custom +} // namespace micro +} // namespace ops + +AllOpsResolver::AllOpsResolver() { + // Please keep this list of Builtin Operators in alphabetical order. + AddAbs(); + AddAdd(); + AddArgMax(); + AddArgMin(); + AddAveragePool2D(); + AddCeil(); + AddConcatenation(); + AddConv2D(); + AddCos(); + AddDepthwiseConv2D(); + AddDequantize(); + AddEqual(); + AddFloor(); + AddFullyConnected(); + AddGreater(); + AddGreaterEqual(); + AddHardSwish(); + AddL2Normalization(); + AddLess(); + AddLessEqual(); + AddLog(); + AddLogicalAnd(); + AddLogicalNot(); + AddLogicalOr(); + AddLogistic(); + AddMaximum(); + AddMaxPool2D(); + AddMean(); + AddMinimum(); + AddMul(); + AddNeg(); + AddNotEqual(); + AddPack(); + AddPad(); + AddPadV2(); + AddPrelu(); + AddQuantize(); + AddReduceMax(); + AddRelu(); + AddRelu6(); + AddReshape(); + AddResizeNearestNeighbor(); + AddRound(); + AddRsqrt(); + AddShape(); + AddSin(); + AddSoftmax(); + AddSplit(); + AddSplitV(); + AddSqrt(); + AddSquare(); + AddStridedSlice(); + AddSub(); + AddSvdf(); + AddTanh(); + AddUnpack(); + + // TODO(b/159644355): Figure out if custom Ops belong in AllOpsResolver. + TfLiteRegistration* registration = + tflite::ops::micro::custom::Register_ETHOSU(); + if (registration) { + AddCustom(tflite::ops::micro::custom::GetString_ETHOSU(), registration); + } +} + +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/all_ops_resolver.h b/code/lib/tfmicro/tensorflow/lite/micro/all_ops_resolver.h similarity index 60% rename from code/lib/tfmicro/tensorflow/lite/micro/kernels/all_ops_resolver.h rename to code/lib/tfmicro/tensorflow/lite/micro/all_ops_resolver.h index 26bb0323..e8105b96 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/all_ops_resolver.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/all_ops_resolver.h @@ -9,17 +9,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_LITE_MICRO_KERNELS_ALL_OPS_RESOLVER_H_ -#define TENSORFLOW_LITE_MICRO_KERNELS_ALL_OPS_RESOLVER_H_ +#ifndef TENSORFLOW_LITE_MICRO_ALL_OPS_RESOLVER_H_ +#define TENSORFLOW_LITE_MICRO_ALL_OPS_RESOLVER_H_ #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/micro_mutable_op_resolver.h" namespace tflite { -namespace ops { -namespace micro { -class AllOpsResolver : public MicroMutableOpResolver { +// The magic number in the template parameter is the maximum number of ops that +// can be added to AllOpsResolver. It can be increased if needed. And most +// applications that care about the memory footprint will want to directly use +// MicroMutableOpResolver and have an application specific template parameter. +// The examples directory has sample code for this. +class AllOpsResolver : public MicroMutableOpResolver<128> { public: AllOpsResolver(); @@ -27,8 +30,6 @@ class AllOpsResolver : public MicroMutableOpResolver { TF_LITE_REMOVE_VIRTUAL_DELETE }; -} // namespace micro -} // namespace ops } // namespace tflite -#endif // TENSORFLOW_LITE_MICRO_KERNELS_ALL_OPS_RESOLVER_H_ +#endif // TENSORFLOW_LITE_MICRO_ALL_OPS_RESOLVER_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.cc b/code/lib/tfmicro/tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.cc new file mode 100644 index 00000000..834f44ca --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.cc @@ -0,0 +1,2898 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h" + +// Keep model aligned to 8 bytes to guarantee aligned 64-bit accesses. +alignas(8) const unsigned char g_keyword_scrambled_model_data[] = { + 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x14, 0x00, 0x10, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x0e, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xd0, 0x6e, 0x00, 0x00, + 0xe4, 0x85, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0xbc, 0x6e, 0x00, 0x00, 0xac, 0x56, 0x00, 0x00, 0x9c, 0x52, 0x00, 0x00, + 0x8c, 0x51, 0x00, 0x00, 0x7c, 0x4d, 0x00, 0x00, 0x2c, 0x4d, 0x00, 0x00, + 0x1c, 0x49, 0x00, 0x00, 0x0c, 0x45, 0x00, 0x00, 0xfc, 0x43, 0x00, 0x00, + 0xec, 0x3f, 0x00, 0x00, 0x9c, 0x3f, 0x00, 0x00, 0x8c, 0x3b, 0x00, 0x00, + 0x7c, 0x37, 0x00, 0x00, 0x6c, 0x36, 0x00, 0x00, 0x5c, 0x32, 0x00, 0x00, + 0x0c, 0x32, 0x00, 0x00, 0xfc, 0x2d, 0x00, 0x00, 0xec, 0x29, 0x00, 0x00, + 0xdc, 0x28, 0x00, 0x00, 0xcc, 0x24, 0x00, 0x00, 0x7c, 0x24, 0x00, 0x00, + 0x6c, 0x22, 0x00, 0x00, 0x5c, 0x1a, 0x00, 0x00, 0xcc, 0x19, 0x00, 0x00, + 0xbc, 0x15, 0x00, 0x00, 0xac, 0x0d, 0x00, 0x00, 0x1c, 0x0d, 0x00, 0x00, + 0x0c, 0x09, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x6c, 0x00, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x2a, 0x91, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x34, 0xe1, 0x4f, 0xa1, + 0x63, 0xa4, 0x62, 0xbf, 0x3e, 0x91, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0xa3, 0xb2, 0x8f, 0xee, 0x35, 0xe6, 0xf2, 0xcc, + 0x68, 0xa0, 0x33, 0xc4, 0x7d, 0x4e, 0xbb, 0xa9, 0x10, 0x32, 0x8e, 0x3d, + 0x76, 0x14, 0x1c, 0x33, 0x0e, 0x77, 0xf7, 0xc8, 0x7b, 0x45, 0xc7, 0xdb, + 0xcf, 0x87, 0xc7, 0x70, 0xa9, 0x29, 0xfd, 0x70, 0x32, 0x96, 0x35, 0x7d, + 0xe9, 0xac, 0x6d, 0x9b, 0xfd, 0xe4, 0xbc, 0x4a, 0x57, 0xcd, 0x43, 0xcc, + 0x73, 0x72, 0xdf, 0x07, 0x68, 0xc5, 0x67, 0xbd, 0x8a, 0x91, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0xb0, 0xfb, 0x5f, 0xdf, + 0x0e, 0xb9, 0xa2, 0xfd, 0x66, 0x86, 0x13, 0x1b, 0x6d, 0x1d, 0x53, 0xdb, + 0x83, 0xbf, 0x44, 0x29, 0x3f, 0x93, 0xee, 0x42, 0x9a, 0xf4, 0x31, 0x6e, + 0xc3, 0x15, 0x7e, 0x48, 0x72, 0x50, 0xc3, 0x53, 0xef, 0x35, 0x1f, 0xc2, + 0x29, 0x42, 0xb4, 0xd7, 0x4b, 0xd7, 0x98, 0x60, 0xb9, 0x3e, 0xbb, 0x31, + 0x35, 0xc3, 0xf6, 0x15, 0x7a, 0x9a, 0x2c, 0xfd, 0xff, 0x04, 0xd9, 0x04, + 0x57, 0x52, 0xae, 0x99, 0xa3, 0x95, 0xae, 0x6a, 0x66, 0x52, 0x5f, 0x91, + 0x17, 0x83, 0x0d, 0x27, 0x16, 0x02, 0x06, 0x64, 0x80, 0x05, 0x99, 0x1c, + 0x6c, 0xab, 0xb1, 0xa1, 0x0e, 0x44, 0x1f, 0x63, 0xe9, 0xc1, 0xab, 0x8d, + 0x08, 0x79, 0x56, 0xe0, 0x90, 0xa5, 0xb8, 0x3b, 0xc4, 0x1e, 0xa5, 0x1f, + 0x64, 0xe4, 0x0b, 0x72, 0x62, 0x19, 0x5f, 0x66, 0xc0, 0x9b, 0x7b, 0xc4, + 0xe5, 0x9f, 0x82, 0xa7, 0x16, 0x92, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x08, 0x00, 0x00, 0x3e, 0x3d, 0xf4, 0x61, 0x45, 0x2a, 0x48, 0x53, + 0x1f, 0x22, 0x74, 0x65, 0xea, 0x5a, 0x00, 0x83, 0x68, 0xf9, 0xbb, 0xa3, + 0xc2, 0x1a, 0x8f, 0xe1, 0xfb, 0x76, 0x6a, 0xe9, 0x1a, 0x0e, 0x4d, 0x32, + 0xc6, 0xf3, 0x8d, 0x85, 0x54, 0xa1, 0xe9, 0xb8, 0x35, 0xee, 0xba, 0x53, + 0x40, 0xa2, 0xea, 0x7f, 0xc3, 0x99, 0x71, 0x17, 0xdd, 0xd5, 0xfe, 0xdf, + 0x5e, 0x15, 0xa0, 0x73, 0xf8, 0x78, 0x49, 0x73, 0xcc, 0xf0, 0x18, 0x12, + 0x06, 0x81, 0xd6, 0x19, 0x2c, 0xa8, 0xd7, 0x80, 0x19, 0x19, 0xbf, 0x1e, + 0x50, 0xb1, 0xfb, 0xb3, 0xa6, 0x56, 0x6f, 0x52, 0xa6, 0xc0, 0xdd, 0x3f, + 0xbb, 0x13, 0x6e, 0x04, 0xdf, 0x79, 0xca, 0x8b, 0xa5, 0x9c, 0xa1, 0x78, + 0x49, 0xca, 0xe5, 0x29, 0xbb, 0x29, 0x7c, 0x96, 0xc6, 0x29, 0x06, 0x99, + 0xec, 0x50, 0xd1, 0xe8, 0x9b, 0xb7, 0x53, 0xd2, 0x36, 0x89, 0xb1, 0x5c, + 0x38, 0xf4, 0x2f, 0xa1, 0xda, 0x6f, 0xd8, 0xd1, 0x62, 0xd2, 0xd4, 0x97, + 0xce, 0xf1, 0xbd, 0x73, 0x2d, 0x92, 0xdb, 0x62, 0x0c, 0xb0, 0x77, 0xed, + 0x32, 0x3a, 0xfc, 0x59, 0x94, 0xef, 0x2b, 0x48, 0x60, 0xb2, 0x82, 0xa2, + 0xb6, 0x51, 0xdb, 0x51, 0x47, 0x99, 0x4c, 0x50, 0x93, 0x53, 0x9d, 0xa9, + 0x3c, 0x94, 0x34, 0x9f, 0xa6, 0x3e, 0x4f, 0x87, 0xd4, 0xa0, 0x40, 0xeb, + 0x7b, 0xfa, 0x1b, 0x7d, 0x03, 0xa8, 0xf8, 0x8b, 0xa5, 0x32, 0x3a, 0xaf, + 0x7e, 0x6b, 0x25, 0x08, 0x97, 0x71, 0x8d, 0x0c, 0x30, 0xc9, 0xa7, 0x23, + 0xe3, 0x51, 0xb3, 0xf2, 0x86, 0xad, 0x12, 0xe2, 0x79, 0x94, 0x7f, 0xf3, + 0xf7, 0x88, 0x67, 0x3e, 0x8e, 0x8e, 0x04, 0x5e, 0x4f, 0x01, 0x6f, 0x1d, + 0x78, 0x42, 0x9e, 0x47, 0x81, 0xdf, 0x03, 0x39, 0x3d, 0x9b, 0xbd, 0xb6, + 0x06, 0x21, 0x82, 0xfe, 0xf2, 0x50, 0xe1, 0x14, 0xbc, 0xe3, 0x5e, 0xe1, + 0xbd, 0x8f, 0xfa, 0x35, 0x31, 0x4e, 0x66, 0xeb, 0x67, 0x49, 0x1c, 0x07, + 0x88, 0xb6, 0x22, 0x0c, 0xeb, 0xd9, 0x9f, 0x9b, 0x8b, 0xe0, 0x9c, 0x3c, + 0xf7, 0x91, 0xab, 0x98, 0x5b, 0x0e, 0x09, 0xdd, 0xe3, 0x0b, 0x14, 0x55, + 0xe9, 0xe4, 0x42, 0xd8, 0xce, 0xd7, 0xfd, 0x4c, 0x20, 0x9f, 0x44, 0x93, + 0xa6, 0x17, 0x8a, 0x68, 0x8f, 0xec, 0x62, 0xd1, 0x97, 0x9c, 0xcc, 0xc4, + 0xd9, 0x42, 0xda, 0xf1, 0x34, 0x04, 0xc6, 0xb6, 0x0f, 0xc7, 0xe6, 0x2d, + 0x26, 0x6e, 0x6f, 0x92, 0x7e, 0xd9, 0xd4, 0x40, 0xc6, 0x70, 0xfa, 0x12, + 0x2a, 0x1b, 0xbc, 0x50, 0xeb, 0x3b, 0x24, 0x96, 0x8d, 0x7c, 0xae, 0xbe, + 0xc3, 0x27, 0xce, 0x97, 0xcf, 0xcd, 0x10, 0x13, 0x01, 0xc6, 0x48, 0x6a, + 0x99, 0x38, 0x79, 0xb9, 0x1c, 0xc9, 0x09, 0xac, 0x96, 0x8c, 0xf7, 0x82, + 0x8f, 0xb8, 0x17, 0x94, 0x2c, 0x5f, 0x40, 0xcc, 0x80, 0xf4, 0x9f, 0xaa, + 0xcb, 0x83, 0x13, 0x7b, 0x3a, 0x78, 0x0a, 0x9f, 0x79, 0x9e, 0xfc, 0x0e, + 0x8f, 0x98, 0x60, 0x39, 0x86, 0x44, 0x8e, 0x4b, 0xc4, 0xad, 0xe6, 0x98, + 0x92, 0x08, 0x84, 0x48, 0x8f, 0x1d, 0x78, 0x10, 0x9e, 0xf7, 0xb8, 0x61, + 0x65, 0x46, 0xdb, 0x4a, 0xcf, 0xc5, 0x37, 0xe3, 0x77, 0x76, 0xcf, 0x0a, + 0x7e, 0x72, 0x3f, 0xe4, 0x51, 0x30, 0x28, 0x57, 0x13, 0xfd, 0xdb, 0x7e, + 0xd6, 0xa3, 0xdd, 0x64, 0xdd, 0x00, 0xd0, 0x7f, 0xbc, 0x48, 0x1d, 0xaf, + 0xde, 0x0e, 0x45, 0xc4, 0xc9, 0xfa, 0xf6, 0xb2, 0xb7, 0x9a, 0x42, 0x8b, + 0x18, 0x08, 0xed, 0xdb, 0xa9, 0xc3, 0x32, 0xf1, 0x9c, 0xcf, 0x16, 0x74, + 0x57, 0xce, 0xe9, 0x44, 0x21, 0xdb, 0x8a, 0x45, 0x89, 0x70, 0x41, 0x5c, + 0xbf, 0x10, 0xdf, 0x83, 0x4a, 0xe4, 0x4c, 0xd8, 0xc9, 0x2e, 0x5b, 0xa3, + 0x05, 0xed, 0x73, 0xb1, 0xb0, 0xb7, 0xc4, 0xd7, 0x0d, 0xea, 0xf6, 0xb4, + 0xc1, 0x5e, 0x12, 0x54, 0x30, 0x73, 0x5c, 0x93, 0xd9, 0xf7, 0xc9, 0x24, + 0x43, 0x8f, 0x4f, 0x8e, 0x94, 0x95, 0xb6, 0xfd, 0xa3, 0x14, 0x42, 0x50, + 0xb8, 0x66, 0xfb, 0xc4, 0xed, 0x72, 0xcf, 0x7b, 0xa9, 0x73, 0xeb, 0xc4, + 0x4a, 0x05, 0xea, 0xb4, 0x47, 0xca, 0x21, 0x56, 0x28, 0xa8, 0x87, 0xb8, + 0x87, 0x0b, 0xe3, 0x8d, 0xfd, 0x70, 0xf7, 0x33, 0x76, 0xf0, 0x3d, 0xa4, + 0x3b, 0x83, 0xab, 0x14, 0x01, 0xe1, 0xb0, 0xa9, 0x44, 0xe8, 0xd7, 0x50, + 0x26, 0x0b, 0xbb, 0x2d, 0x57, 0x39, 0x82, 0x7c, 0x71, 0xd8, 0x12, 0xaf, + 0xf3, 0x9f, 0x46, 0xbd, 0x62, 0xd6, 0x61, 0xf5, 0xb7, 0x04, 0x94, 0xbf, + 0x87, 0xea, 0xc4, 0xc4, 0x33, 0xcf, 0x36, 0x3b, 0x4f, 0xc7, 0x71, 0xf1, + 0x98, 0xe6, 0xb0, 0x96, 0x25, 0xd7, 0xac, 0x75, 0xfc, 0x92, 0xe0, 0x69, + 0x72, 0x37, 0x8d, 0x40, 0x31, 0xaa, 0x2c, 0x86, 0xfb, 0x95, 0x3f, 0x9c, + 0x23, 0xd4, 0x39, 0x99, 0xff, 0xea, 0x95, 0x79, 0xb9, 0x2e, 0xb0, 0x33, + 0xf1, 0xe8, 0xd0, 0x42, 0xb5, 0x70, 0x5c, 0xca, 0x69, 0x48, 0x28, 0x23, + 0x58, 0xb4, 0x07, 0xfc, 0x3e, 0x15, 0x29, 0x00, 0xa9, 0x22, 0x44, 0x70, + 0xd0, 0xc7, 0x01, 0x0d, 0x3e, 0xfc, 0x57, 0xb7, 0x54, 0x3a, 0xc3, 0x43, + 0xd6, 0x2f, 0x55, 0x09, 0x52, 0x4a, 0x6b, 0x8e, 0x4c, 0x82, 0xbb, 0x4e, + 0x3e, 0x38, 0xe1, 0x9e, 0x72, 0x83, 0xec, 0x40, 0xf5, 0xf7, 0x0e, 0x3c, + 0x24, 0xed, 0xda, 0xf2, 0x39, 0x6c, 0xad, 0xeb, 0xff, 0xfb, 0x4a, 0x38, + 0x50, 0x49, 0x28, 0x3d, 0x05, 0xb2, 0x98, 0x44, 0x2b, 0x61, 0xa2, 0x9b, + 0x3a, 0x3c, 0xad, 0xd9, 0x8c, 0xef, 0x3c, 0x72, 0x50, 0x74, 0x13, 0x80, + 0xc4, 0x7e, 0x6e, 0xf3, 0xc9, 0xdf, 0x63, 0xf6, 0x41, 0xb2, 0x08, 0x78, + 0x9b, 0x7c, 0xa9, 0x13, 0xd1, 0x21, 0xe7, 0x5e, 0x6a, 0x0d, 0x64, 0xf7, + 0x52, 0x75, 0xf2, 0x80, 0x69, 0xbe, 0x43, 0xf8, 0xd4, 0xad, 0x49, 0xfc, + 0x97, 0x76, 0x1c, 0xb6, 0x43, 0x9e, 0xcb, 0x45, 0x4d, 0x75, 0x07, 0xae, + 0xdb, 0xbf, 0xf5, 0x8a, 0xeb, 0xb9, 0x6b, 0x12, 0x06, 0xbf, 0x94, 0xad, + 0x77, 0x29, 0xb1, 0xae, 0x24, 0x9b, 0x4d, 0xdc, 0xe1, 0x5e, 0xd7, 0x57, + 0xec, 0xd1, 0xd8, 0xad, 0xf0, 0x06, 0x08, 0x43, 0x33, 0x99, 0xd2, 0x04, + 0xfc, 0xc8, 0xf6, 0x53, 0x3d, 0x73, 0xd4, 0x36, 0xd3, 0x8e, 0x4a, 0xcd, + 0xb1, 0xe9, 0xcb, 0x3a, 0x5f, 0x54, 0xbc, 0xde, 0x16, 0xa2, 0x85, 0xde, + 0x35, 0x27, 0x99, 0x32, 0x4f, 0xb9, 0x2c, 0x16, 0xa2, 0x6e, 0xae, 0x75, + 0x60, 0x77, 0xe9, 0x08, 0x0f, 0x08, 0xc4, 0xd0, 0x62, 0xc7, 0xd2, 0x1f, + 0x3b, 0x29, 0xdd, 0xb7, 0xea, 0xa3, 0x58, 0xaf, 0x4c, 0x05, 0xd2, 0x82, + 0x6a, 0xe0, 0xc4, 0xe9, 0x70, 0x7e, 0xf2, 0xca, 0x82, 0x6a, 0xae, 0xc1, + 0x9a, 0x42, 0x5d, 0x46, 0x4a, 0xb7, 0x8f, 0x4d, 0x33, 0xfe, 0x6f, 0x47, + 0xb5, 0x49, 0xb3, 0x89, 0x51, 0x31, 0x74, 0x68, 0x14, 0xda, 0x0a, 0x41, + 0x3d, 0x1f, 0x8e, 0x30, 0x8c, 0x77, 0xd1, 0xa9, 0x36, 0x41, 0x78, 0x34, + 0xb7, 0x7e, 0x4e, 0x7a, 0x77, 0x12, 0x43, 0x97, 0x43, 0xba, 0xd6, 0x28, + 0x14, 0x2a, 0x9f, 0x98, 0xb4, 0x39, 0x08, 0x5c, 0xb7, 0xb8, 0x03, 0x63, + 0x62, 0x68, 0xc6, 0x9a, 0x4d, 0xf5, 0xdc, 0x7c, 0x0f, 0x7e, 0x77, 0xdc, + 0x85, 0x53, 0x31, 0x8c, 0x53, 0x8b, 0x27, 0xc4, 0xb7, 0x3d, 0xd0, 0x94, + 0x9b, 0x7e, 0x59, 0x59, 0x03, 0x09, 0x8c, 0x30, 0x70, 0x7d, 0x9c, 0x73, + 0x89, 0x6c, 0x5f, 0xbf, 0xf9, 0xc7, 0x72, 0x76, 0x12, 0x98, 0xe3, 0xbe, + 0xc3, 0x67, 0xdf, 0xa1, 0x76, 0xa3, 0xec, 0x44, 0x30, 0x70, 0x2f, 0x6a, + 0x86, 0x28, 0xb9, 0x9d, 0x7f, 0x93, 0xf2, 0x4a, 0x34, 0x48, 0x1f, 0x2e, + 0x2e, 0x95, 0x88, 0xdb, 0x1f, 0x2c, 0x19, 0x46, 0x2e, 0x91, 0x5f, 0x81, + 0x0d, 0x08, 0x9d, 0x03, 0x0b, 0xaf, 0x59, 0x0a, 0x41, 0xad, 0x4d, 0x6c, + 0x09, 0x0e, 0x9f, 0xd1, 0xc4, 0xdb, 0xac, 0x59, 0x27, 0x04, 0x1c, 0x73, + 0xe9, 0xf3, 0xe8, 0x54, 0xd9, 0x11, 0x31, 0xb2, 0xed, 0x2d, 0x8c, 0xeb, + 0x99, 0x26, 0x48, 0x9e, 0xac, 0x88, 0x96, 0xcb, 0x19, 0x49, 0xfa, 0x4a, + 0x82, 0xd5, 0x5d, 0xb8, 0x0f, 0x22, 0x3f, 0xb6, 0x5c, 0x02, 0x2a, 0xb9, + 0xd9, 0xfe, 0x4d, 0x9d, 0xdb, 0x85, 0x90, 0x19, 0x7f, 0x1a, 0x44, 0xa3, + 0x74, 0x68, 0xbf, 0xa2, 0x3b, 0xb4, 0x3b, 0xeb, 0xab, 0x99, 0xc2, 0x46, + 0x50, 0x7e, 0xec, 0xa9, 0xb4, 0x86, 0xfa, 0x50, 0xcb, 0x71, 0x7e, 0x75, + 0xa5, 0xca, 0xa6, 0x2f, 0x40, 0x1d, 0xa1, 0x4a, 0x5c, 0x91, 0xd7, 0x2a, + 0xa6, 0x17, 0x11, 0x4d, 0x19, 0x2b, 0xb3, 0x0f, 0xf0, 0xb3, 0x06, 0x70, + 0x51, 0x5c, 0x52, 0x8c, 0xdf, 0xe3, 0x19, 0x92, 0x08, 0x40, 0xa2, 0xb4, + 0xc0, 0xf2, 0xe8, 0x44, 0xcc, 0x36, 0xaa, 0xf9, 0xf8, 0xfc, 0x2d, 0x83, + 0x79, 0xc6, 0x58, 0xc1, 0xdf, 0x32, 0xb7, 0xde, 0x0f, 0x3e, 0xc0, 0xa8, + 0x7e, 0xeb, 0xf2, 0x30, 0x16, 0xdf, 0x38, 0xcb, 0x69, 0xd9, 0x44, 0x0d, + 0x44, 0xf4, 0x45, 0x9c, 0x81, 0xc8, 0xe7, 0x06, 0xae, 0x95, 0xaf, 0xff, + 0x17, 0x3b, 0x1c, 0x3f, 0xda, 0xa5, 0xf8, 0xfd, 0x9c, 0xf1, 0x0a, 0xca, + 0xda, 0xc0, 0xfa, 0x02, 0xc4, 0xce, 0x78, 0xfb, 0x35, 0x8c, 0xfe, 0x55, + 0xad, 0x0d, 0x9b, 0xeb, 0x10, 0xf1, 0x7b, 0xb1, 0x09, 0xf8, 0xef, 0xfc, + 0xde, 0x7a, 0x69, 0x74, 0x76, 0xef, 0x91, 0x64, 0x33, 0xc4, 0x08, 0x15, + 0x73, 0x85, 0x56, 0xae, 0x9c, 0xf6, 0xdd, 0x55, 0x19, 0x96, 0xe6, 0x41, + 0x12, 0xc9, 0x87, 0x91, 0x9e, 0xc6, 0x18, 0xe8, 0xbf, 0xa0, 0x59, 0xfd, + 0x20, 0xab, 0xb5, 0xcf, 0x0f, 0x6e, 0x30, 0xd3, 0xc5, 0x70, 0xf2, 0x50, + 0xa4, 0x2a, 0xdf, 0xb0, 0x45, 0xfc, 0x82, 0x1a, 0x3b, 0xfe, 0x0c, 0xad, + 0x41, 0x95, 0xf1, 0xd6, 0x85, 0xa2, 0xc9, 0xff, 0xbe, 0x3a, 0x64, 0x70, + 0x43, 0xc0, 0xc5, 0xc8, 0x80, 0x11, 0x0d, 0x20, 0xcd, 0xf2, 0xa2, 0xbb, + 0x43, 0x68, 0x0e, 0xf4, 0x01, 0xb3, 0x73, 0x79, 0x9f, 0x68, 0x41, 0x63, + 0x3e, 0xda, 0xf9, 0xf4, 0x23, 0x57, 0x97, 0x84, 0x99, 0xe8, 0x5e, 0xdb, + 0xaa, 0x24, 0xab, 0x9c, 0x40, 0x83, 0xf9, 0x3f, 0x4f, 0x5a, 0x53, 0xa6, + 0xf1, 0xe8, 0x95, 0xcf, 0xcb, 0x50, 0x13, 0x51, 0xa7, 0x8c, 0x71, 0x1d, + 0xff, 0xcc, 0x66, 0xab, 0xff, 0xca, 0xc5, 0xc3, 0x73, 0x45, 0xb7, 0x21, + 0x1d, 0x65, 0x7a, 0xe5, 0x1f, 0x3f, 0x1a, 0x58, 0x23, 0x28, 0xc8, 0xf3, + 0xbf, 0x98, 0x25, 0xc0, 0x83, 0x68, 0xf0, 0x62, 0x63, 0x90, 0xcf, 0x1f, + 0x20, 0xb8, 0x04, 0x5c, 0xc4, 0x80, 0x5b, 0xf4, 0x6d, 0xdc, 0xe9, 0xac, + 0xd8, 0x13, 0x3b, 0x42, 0xf8, 0x4e, 0xa2, 0x1c, 0xce, 0x3f, 0x8d, 0x15, + 0xd3, 0x87, 0x1b, 0x44, 0x79, 0x52, 0x34, 0x4b, 0x63, 0x4d, 0xbf, 0x95, + 0xec, 0xae, 0xf9, 0xc6, 0x7b, 0x7b, 0x85, 0x8c, 0x4f, 0x20, 0x58, 0x9d, + 0x48, 0x03, 0x2f, 0x77, 0x2e, 0x8b, 0x6f, 0x66, 0x76, 0xb9, 0xb8, 0xb7, + 0x34, 0x5a, 0x63, 0x06, 0x85, 0x82, 0x5f, 0x23, 0x8f, 0x8d, 0x0c, 0x92, + 0x3b, 0xd2, 0x8a, 0x1b, 0x39, 0xee, 0x6a, 0xbc, 0xf6, 0x94, 0x2a, 0xc6, + 0x73, 0xa6, 0x99, 0x98, 0xdc, 0x96, 0xd7, 0xc1, 0xfe, 0x9b, 0xc8, 0xfb, + 0x86, 0x5a, 0xad, 0xce, 0xf8, 0xd5, 0x32, 0x62, 0x96, 0x63, 0xaf, 0x4c, + 0x4a, 0xae, 0xec, 0x26, 0x3d, 0x84, 0x69, 0x50, 0x5f, 0x37, 0x9b, 0x29, + 0xac, 0x15, 0x76, 0x3d, 0x33, 0x96, 0x06, 0xde, 0xc1, 0x6d, 0xa2, 0xc7, + 0xc3, 0x8a, 0x20, 0x2e, 0xf7, 0x08, 0x55, 0x83, 0x23, 0x9c, 0x23, 0x2d, + 0x3a, 0xa1, 0x32, 0xbc, 0x47, 0x48, 0xd5, 0x6a, 0x71, 0xb9, 0xcc, 0x2d, + 0x99, 0xa0, 0x37, 0x07, 0x46, 0x45, 0xbe, 0xf0, 0x27, 0x5a, 0x25, 0x72, + 0x58, 0x47, 0x6d, 0xbf, 0x23, 0xdc, 0x48, 0x44, 0x45, 0x95, 0xb1, 0x62, + 0xf1, 0x7e, 0x4c, 0x95, 0x1c, 0xb4, 0x17, 0x8b, 0x59, 0x2e, 0xf3, 0x4f, + 0x45, 0x3b, 0x5d, 0x67, 0x92, 0x52, 0xd8, 0xc1, 0x91, 0xfa, 0x53, 0xaa, + 0x87, 0xc0, 0xa7, 0xb0, 0x9f, 0x10, 0xe8, 0xac, 0x45, 0x52, 0xbb, 0x17, + 0xee, 0xf6, 0x18, 0xbe, 0x02, 0x70, 0xce, 0x79, 0x66, 0x72, 0xf9, 0xf6, + 0xca, 0x66, 0xff, 0xa4, 0x9a, 0xd9, 0xb7, 0x07, 0xa9, 0xc1, 0x23, 0x7e, + 0x7b, 0x9c, 0xe3, 0x02, 0x7a, 0xcc, 0xa3, 0x67, 0xb7, 0xb0, 0x37, 0xba, + 0xae, 0x12, 0xda, 0x48, 0x6e, 0x7f, 0xde, 0x5f, 0x75, 0x15, 0xca, 0xd2, + 0x46, 0xdd, 0xb0, 0x82, 0xbf, 0x6d, 0xe9, 0x51, 0x66, 0xa5, 0x9e, 0x0c, + 0xd5, 0x03, 0xbd, 0x97, 0x0e, 0x1b, 0x88, 0xf6, 0x61, 0x5a, 0x8b, 0xe0, + 0xdd, 0x3e, 0x59, 0x4c, 0x35, 0xfd, 0xb0, 0x3b, 0x79, 0x8c, 0x1c, 0x96, + 0x97, 0x35, 0x62, 0x36, 0x62, 0x4c, 0x4b, 0x46, 0xb1, 0x21, 0xf7, 0xf0, + 0x34, 0xdc, 0xd9, 0x9f, 0xf8, 0x53, 0x7d, 0xca, 0xbc, 0x4d, 0xaf, 0xf4, + 0xb7, 0x2f, 0xa7, 0x5d, 0x18, 0xf9, 0x3b, 0xa9, 0xb0, 0xbb, 0xdf, 0xfa, + 0x28, 0x2b, 0x58, 0xce, 0x46, 0x01, 0x3f, 0x76, 0xf2, 0x39, 0x45, 0x8b, + 0x3c, 0xda, 0x62, 0x2b, 0x6b, 0xe1, 0x5f, 0x14, 0xfc, 0x79, 0x17, 0x2d, + 0xe2, 0xe5, 0x8c, 0xc5, 0xde, 0x91, 0xfd, 0xf5, 0x6d, 0x9b, 0x6b, 0xbb, + 0xb0, 0x13, 0xae, 0xbe, 0x1e, 0xa8, 0x8f, 0x3c, 0xfd, 0x24, 0xbe, 0xb8, + 0x39, 0x80, 0x03, 0x06, 0x8b, 0xff, 0xca, 0x90, 0x88, 0x0f, 0x45, 0xc4, + 0xeb, 0x50, 0x52, 0xf5, 0x00, 0x8c, 0x16, 0x9d, 0x26, 0xaa, 0xec, 0xb1, + 0x44, 0xd6, 0xfe, 0x67, 0xa3, 0xc1, 0xec, 0x4a, 0x12, 0xa6, 0x7c, 0x7c, + 0xc3, 0x46, 0x1c, 0x64, 0x61, 0x67, 0xec, 0xce, 0x1e, 0xa2, 0xb4, 0xdd, + 0x6e, 0x7f, 0x02, 0x14, 0xf4, 0x1c, 0x17, 0xa7, 0x31, 0x9f, 0xc2, 0xc6, + 0xc0, 0x21, 0x41, 0x88, 0x61, 0xd8, 0xca, 0x06, 0xa5, 0xe4, 0xef, 0xa4, + 0xaa, 0x4d, 0xa3, 0xad, 0x5f, 0xd4, 0x0c, 0x6b, 0x14, 0x38, 0x2e, 0xe8, + 0x87, 0x5a, 0x68, 0x10, 0x51, 0xd8, 0xbb, 0xa6, 0xd9, 0xdc, 0xd3, 0x7f, + 0x1f, 0xea, 0xa8, 0xcc, 0x3f, 0x43, 0xa4, 0x04, 0x95, 0xb4, 0xde, 0x2f, + 0x07, 0x5d, 0x91, 0x1c, 0x8e, 0xc3, 0xbc, 0xaa, 0x46, 0x8a, 0xa8, 0x42, + 0xa7, 0x2c, 0x0f, 0x1f, 0xb3, 0xe2, 0x8a, 0x0b, 0xa0, 0x3f, 0xfb, 0x87, + 0x9e, 0x42, 0xa5, 0x60, 0xce, 0x5a, 0x54, 0x91, 0x26, 0x51, 0xea, 0x81, + 0x6f, 0xf1, 0x54, 0x93, 0xe7, 0xa0, 0xf8, 0x64, 0xab, 0x1d, 0x0d, 0x9d, + 0x64, 0x6a, 0xd5, 0x19, 0x03, 0xbb, 0x94, 0x7f, 0x0a, 0xb8, 0x6b, 0x87, + 0xc3, 0x1a, 0x38, 0xe5, 0xe8, 0xba, 0x13, 0x17, 0xeb, 0x13, 0xcc, 0xac, + 0xcb, 0x1f, 0x96, 0x4c, 0x3b, 0x18, 0xfb, 0xe8, 0x5c, 0x54, 0xce, 0x1a, + 0x91, 0x44, 0xf5, 0x49, 0x6c, 0x38, 0x2a, 0x92, 0x8a, 0x0d, 0x3d, 0x08, + 0xc2, 0x5f, 0x6c, 0xac, 0x48, 0xb3, 0xdc, 0x2e, 0xa6, 0x5a, 0xa8, 0xee, + 0x22, 0x9a, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x96, 0xc5, 0x3a, 0x4e, 0x42, 0x7d, 0x27, 0xce, 0x44, 0x84, 0xf1, 0x67, + 0x8c, 0xc5, 0xdd, 0x75, 0x3b, 0x8a, 0xed, 0x2e, 0x29, 0x62, 0x7b, 0xb0, + 0xe6, 0xa3, 0xb4, 0x61, 0x73, 0x10, 0xff, 0x0e, 0x0c, 0x98, 0x74, 0xef, + 0xbb, 0xc4, 0xca, 0x03, 0x88, 0xa4, 0x96, 0x61, 0xef, 0x36, 0x6d, 0xa2, + 0xb1, 0xc8, 0xf0, 0xac, 0xf1, 0xb2, 0x08, 0x56, 0xc7, 0x99, 0xcf, 0xae, + 0x0a, 0x37, 0x85, 0x60, 0x78, 0x2d, 0x14, 0xda, 0xb1, 0xa7, 0x00, 0xb6, + 0x00, 0x04, 0x76, 0x80, 0x0e, 0x9f, 0x2a, 0x30, 0x8b, 0x85, 0xd9, 0xc1, + 0xaf, 0xee, 0x27, 0x80, 0x20, 0xed, 0xef, 0x25, 0x5c, 0x98, 0x6b, 0xcc, + 0xf8, 0x72, 0xfb, 0x3f, 0x13, 0xe6, 0x9b, 0x47, 0xee, 0xa1, 0x18, 0x55, + 0xa0, 0x68, 0xbe, 0xd4, 0x21, 0x59, 0x72, 0xa8, 0xa4, 0xd2, 0x33, 0x57, + 0x50, 0xfc, 0x6b, 0xa8, 0x49, 0x1b, 0x74, 0xdb, 0x5a, 0x16, 0xb8, 0x52, + 0x0c, 0xda, 0xa0, 0xa3, 0xff, 0x33, 0x56, 0x82, 0x0f, 0x0a, 0x90, 0x82, + 0xee, 0xf1, 0x1b, 0xb3, 0x05, 0x44, 0x39, 0x01, 0xf7, 0x1e, 0xff, 0xcb, + 0xea, 0xd0, 0xb6, 0x20, 0xbc, 0x84, 0xb1, 0xf9, 0xa2, 0xc1, 0x56, 0xe6, + 0xfa, 0x47, 0xc9, 0xfd, 0x45, 0x77, 0x51, 0x8e, 0x01, 0xe4, 0x17, 0x20, + 0x6f, 0x99, 0xe3, 0x90, 0x2f, 0xcc, 0xaf, 0xd9, 0x61, 0x32, 0x91, 0x62, + 0x58, 0xf4, 0x98, 0xf5, 0xf4, 0xeb, 0x13, 0xeb, 0xdc, 0x8a, 0xac, 0xb2, + 0x9e, 0xcf, 0xe7, 0xa7, 0xd4, 0x97, 0x22, 0x12, 0x08, 0x10, 0x6d, 0x40, + 0xea, 0x26, 0xea, 0x42, 0x29, 0x6e, 0x75, 0x62, 0x47, 0x08, 0x17, 0xa8, + 0x69, 0x0f, 0xf7, 0x35, 0x59, 0x23, 0x86, 0x83, 0xfd, 0xb5, 0x61, 0x98, + 0x9c, 0x4d, 0x37, 0xda, 0x9f, 0xfc, 0xfb, 0x16, 0xb7, 0x6c, 0x52, 0xee, + 0xa8, 0x9c, 0x3e, 0x93, 0x43, 0xc5, 0x2b, 0xd4, 0xd0, 0x9f, 0x69, 0x2c, + 0xc9, 0x1f, 0x2e, 0xdf, 0x5b, 0xe6, 0xc6, 0x5f, 0x71, 0xd1, 0xd7, 0xb2, + 0x8f, 0x3a, 0xba, 0x60, 0x75, 0x3d, 0x34, 0x41, 0x43, 0x9b, 0x13, 0xc0, + 0x3b, 0x30, 0xc5, 0xe9, 0x84, 0x81, 0xde, 0x85, 0x4e, 0x65, 0x7b, 0x21, + 0x37, 0xb8, 0xef, 0x24, 0x19, 0xaa, 0x26, 0x0c, 0x27, 0xa7, 0xd9, 0x29, + 0x47, 0x1a, 0x15, 0x42, 0x1e, 0x30, 0x79, 0x79, 0x96, 0x09, 0x62, 0x26, + 0xad, 0x98, 0x8b, 0xcb, 0x3d, 0xeb, 0x66, 0x83, 0x77, 0xd9, 0x79, 0x4d, + 0x05, 0x81, 0x72, 0xe9, 0xe0, 0x6f, 0x13, 0x00, 0x7e, 0xa3, 0x92, 0x82, + 0x1c, 0x90, 0x83, 0x4b, 0x15, 0x97, 0x0f, 0x92, 0xe2, 0xd3, 0x3d, 0xd7, + 0x6c, 0xb9, 0x60, 0x9a, 0x23, 0x52, 0xbe, 0x59, 0xc9, 0x36, 0x9e, 0xf7, + 0x77, 0x09, 0x79, 0x01, 0xcc, 0xec, 0x17, 0xd1, 0x74, 0xbc, 0x58, 0x65, + 0x45, 0x3c, 0x86, 0xf1, 0xbc, 0xbd, 0x95, 0x54, 0x46, 0x45, 0x7b, 0x4c, + 0xa2, 0xea, 0x2a, 0x6e, 0xa8, 0xd1, 0x66, 0x03, 0xb2, 0x6a, 0xe0, 0xd3, + 0x07, 0x8d, 0xe0, 0x09, 0x81, 0x42, 0xe3, 0x97, 0xc4, 0xe7, 0x37, 0xc5, + 0x82, 0xcf, 0xb1, 0xec, 0xba, 0xbd, 0xf4, 0xb6, 0x41, 0xb2, 0xb8, 0xa6, + 0x3a, 0x85, 0x4b, 0x4f, 0x46, 0x48, 0xe9, 0x9b, 0x72, 0xf5, 0xb0, 0x64, + 0x66, 0x75, 0x42, 0xb4, 0x00, 0xbe, 0x11, 0x6d, 0x86, 0x93, 0x07, 0x50, + 0xa7, 0xef, 0x55, 0x42, 0xcf, 0xe8, 0x61, 0xd0, 0x9b, 0x11, 0x84, 0x8c, + 0x74, 0xe4, 0xb8, 0x3f, 0x48, 0xb3, 0x61, 0xe3, 0xea, 0x66, 0x86, 0x94, + 0x95, 0x12, 0x77, 0x26, 0x75, 0x30, 0xb5, 0xd3, 0x7a, 0xad, 0x2d, 0x58, + 0x46, 0x1b, 0x4b, 0xd9, 0x2d, 0x1e, 0x0b, 0xff, 0xd7, 0x03, 0x56, 0x3b, + 0xbd, 0x65, 0xb0, 0xf9, 0xfe, 0x43, 0x1c, 0x9c, 0x18, 0x82, 0x78, 0x5e, + 0x06, 0x02, 0x21, 0x70, 0xb2, 0x7f, 0xb5, 0x63, 0x71, 0x85, 0x95, 0x79, + 0xae, 0x1e, 0xc6, 0x62, 0x7a, 0x7c, 0x63, 0x46, 0x70, 0x1c, 0x58, 0x72, + 0x1d, 0xde, 0xca, 0xb4, 0xfc, 0xc8, 0x56, 0x38, 0x32, 0xf4, 0x0b, 0x56, + 0x87, 0x6b, 0x5b, 0x53, 0xd2, 0x2c, 0x35, 0xef, 0x5b, 0x33, 0x59, 0x13, + 0x76, 0x82, 0x30, 0x80, 0x23, 0x10, 0x07, 0x4c, 0x3f, 0xac, 0x9c, 0x58, + 0x2d, 0x04, 0xe6, 0x6a, 0xd3, 0x5c, 0xf9, 0xb6, 0x59, 0x4e, 0x85, 0xfe, + 0x01, 0x71, 0xf0, 0xf7, 0xf2, 0x1f, 0x46, 0xd5, 0x20, 0x3c, 0x9b, 0xc2, + 0x1e, 0x73, 0x1c, 0x56, 0x9c, 0x76, 0x8c, 0x12, 0x95, 0x51, 0xd4, 0x6f, + 0x5b, 0x3a, 0xa7, 0x5f, 0xa7, 0xe4, 0xfa, 0xb7, 0x1a, 0xdd, 0xb6, 0x4c, + 0x01, 0x02, 0xae, 0x9c, 0x02, 0x0d, 0x66, 0x2f, 0x40, 0x87, 0xa1, 0xbc, + 0xf3, 0xde, 0xf4, 0xdb, 0x65, 0xee, 0xcc, 0xca, 0xe1, 0x7a, 0xa2, 0xf4, + 0xf7, 0xf5, 0x7c, 0x2a, 0x3f, 0xa4, 0x67, 0xbb, 0x07, 0x50, 0x7a, 0x29, + 0x8a, 0xcf, 0x2c, 0x7a, 0x0e, 0x0d, 0xc7, 0x95, 0x8b, 0xf4, 0xe2, 0x50, + 0xe1, 0xc1, 0x40, 0x16, 0x99, 0x5c, 0x72, 0xe7, 0xe4, 0x01, 0xeb, 0x29, + 0x6a, 0x99, 0xf2, 0x67, 0x23, 0x46, 0x1f, 0xaa, 0xea, 0xc1, 0x51, 0x30, + 0xeb, 0x7d, 0x34, 0x52, 0x91, 0x37, 0x2d, 0xc6, 0x5c, 0x3a, 0x7c, 0x54, + 0xc0, 0x79, 0xdc, 0xf9, 0xbf, 0x08, 0x2a, 0xf6, 0xe1, 0x1e, 0xee, 0xc6, + 0xd2, 0xe9, 0x30, 0x27, 0x60, 0x0c, 0xa2, 0x63, 0x16, 0x06, 0x3d, 0xe2, + 0xf5, 0x6f, 0xea, 0xe4, 0x4d, 0x9f, 0x2d, 0x36, 0x62, 0x95, 0x47, 0x5d, + 0x00, 0x22, 0x9f, 0x0c, 0xbb, 0x71, 0xad, 0xea, 0xe7, 0x62, 0x59, 0x21, + 0xd1, 0xaf, 0x04, 0x5a, 0xfc, 0x1f, 0x28, 0x6b, 0x6f, 0x71, 0xec, 0xd4, + 0xbd, 0x9c, 0x88, 0xfb, 0x3f, 0x04, 0xea, 0xd6, 0xb2, 0x24, 0xe5, 0x28, + 0xfe, 0xc5, 0x3e, 0x15, 0x00, 0x8c, 0xa2, 0xdf, 0x18, 0x3d, 0x10, 0x9a, + 0xb1, 0xcd, 0x64, 0xda, 0x87, 0x41, 0xc8, 0xa1, 0x1c, 0x97, 0xd5, 0x44, + 0xd9, 0x51, 0xd2, 0x96, 0xed, 0xad, 0x28, 0x1f, 0x03, 0x89, 0x21, 0xbd, + 0x79, 0x91, 0x48, 0x9c, 0x8e, 0x17, 0xfd, 0x36, 0x72, 0xf6, 0x69, 0x4f, + 0x3f, 0x02, 0x57, 0xcc, 0x3f, 0x1c, 0x49, 0x82, 0x00, 0x45, 0x9e, 0x29, + 0x83, 0x14, 0x12, 0xbb, 0xd2, 0xd0, 0x1a, 0x66, 0x0f, 0x57, 0x24, 0xd4, + 0x9f, 0x46, 0x0c, 0xf4, 0xb8, 0x28, 0x85, 0x52, 0xe2, 0xa1, 0xc2, 0x3a, + 0x8c, 0x34, 0x4a, 0x81, 0xe3, 0xbc, 0xa2, 0x67, 0x67, 0x12, 0x13, 0xc4, + 0xe7, 0xd7, 0x2c, 0x4e, 0xa9, 0xf5, 0xed, 0x63, 0xf2, 0x18, 0x9c, 0x0c, + 0xe2, 0x4d, 0x25, 0x23, 0x30, 0x3e, 0x49, 0x29, 0xa6, 0x37, 0xdf, 0xc2, + 0xdc, 0xf6, 0x5e, 0xae, 0x45, 0xd7, 0x8d, 0x56, 0xba, 0x29, 0x4f, 0xee, + 0xc9, 0x26, 0xd7, 0xbf, 0x10, 0x4d, 0x0a, 0x3b, 0x3d, 0x1f, 0xd5, 0x72, + 0xe1, 0xe6, 0xf5, 0x23, 0x4a, 0x17, 0x2d, 0xe4, 0x40, 0x55, 0x9b, 0x39, + 0x66, 0x36, 0xe4, 0x6d, 0x6d, 0xb6, 0x8d, 0x2a, 0x7e, 0x76, 0x73, 0xa5, + 0x86, 0x20, 0x3d, 0x18, 0xa0, 0x6c, 0x35, 0x59, 0xc8, 0x1c, 0xef, 0x0f, + 0x36, 0x1d, 0x6f, 0xba, 0x89, 0xb9, 0x9e, 0x7a, 0x58, 0x1d, 0x43, 0xad, + 0x85, 0x8b, 0x6b, 0xcc, 0x25, 0xb8, 0xe4, 0xdd, 0xa1, 0x35, 0xd9, 0xef, + 0xc4, 0xb1, 0xf6, 0x99, 0x27, 0x17, 0xb7, 0xbe, 0xd1, 0x4f, 0xa1, 0x81, + 0x4e, 0xb6, 0x19, 0xcd, 0xa0, 0x92, 0xeb, 0x56, 0x41, 0x4f, 0x37, 0xca, + 0x3b, 0x43, 0x85, 0x86, 0xdf, 0x5d, 0x5a, 0x8c, 0xd4, 0x5b, 0xc4, 0x28, + 0xdb, 0x16, 0xea, 0x3a, 0x2e, 0x9e, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0xea, 0x59, 0x40, 0xc4, 0x40, 0x8b, 0x6a, 0x8a, + 0xb8, 0x7f, 0x1e, 0x0b, 0xfe, 0xab, 0xa4, 0xac, 0x42, 0x91, 0xc5, 0xfa, + 0x2c, 0x7e, 0xb4, 0xf9, 0x5c, 0xd5, 0x4c, 0x6a, 0x74, 0x82, 0x90, 0x81, + 0x96, 0xb0, 0xf4, 0xd4, 0xba, 0xc9, 0xa3, 0x2e, 0x26, 0x0a, 0xc9, 0x55, + 0x65, 0xac, 0xde, 0x83, 0x37, 0xec, 0x0e, 0xf6, 0xdc, 0x8c, 0x34, 0xe6, + 0x57, 0xde, 0x32, 0x0a, 0x02, 0x62, 0x4f, 0x6a, 0x92, 0xa5, 0xb4, 0x40, + 0xde, 0x57, 0xf4, 0xd1, 0xa3, 0x1c, 0xd3, 0xf7, 0x4a, 0x15, 0xcc, 0x27, + 0x26, 0x00, 0xba, 0xf3, 0xfa, 0x4e, 0xc6, 0xe9, 0xc3, 0x05, 0x3d, 0x3a, + 0x89, 0x96, 0x7d, 0x41, 0xac, 0xca, 0x28, 0x7f, 0x69, 0x02, 0x40, 0x03, + 0x93, 0x86, 0x85, 0x85, 0x73, 0x00, 0x09, 0x5a, 0xcf, 0x5f, 0x1d, 0xaa, + 0x46, 0x41, 0x9d, 0x08, 0xbf, 0xea, 0x45, 0x9b, 0x93, 0xda, 0x9e, 0x81, + 0xba, 0x9e, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, + 0x6a, 0x1f, 0x9b, 0x03, 0xdd, 0xe4, 0x16, 0x07, 0x7f, 0x5b, 0xb0, 0xee, + 0xac, 0x55, 0xc4, 0x50, 0xe6, 0x2b, 0x17, 0xed, 0x7f, 0x50, 0x4d, 0x71, + 0x73, 0xae, 0xe0, 0x4d, 0xce, 0x08, 0xd9, 0x8b, 0x83, 0x2c, 0x01, 0x48, + 0x02, 0xd3, 0xbb, 0xca, 0x86, 0xd7, 0xca, 0x5f, 0xc7, 0xce, 0x59, 0xdf, + 0xc1, 0xcc, 0xf7, 0x7b, 0x54, 0xf8, 0x0d, 0x4f, 0x81, 0x9e, 0x50, 0x6a, + 0x65, 0x66, 0x4a, 0xec, 0x7a, 0x1b, 0x92, 0xb2, 0x39, 0x8f, 0x5d, 0x41, + 0x33, 0xcf, 0xe6, 0x1b, 0x34, 0x5d, 0xe1, 0xf6, 0xef, 0xcb, 0xa0, 0x55, + 0x7e, 0x1f, 0x45, 0x38, 0xb9, 0x56, 0x15, 0x3b, 0x70, 0xab, 0xc8, 0x2f, + 0x1c, 0xb9, 0x7d, 0x37, 0xe1, 0xb4, 0x03, 0x44, 0x5a, 0xf6, 0x57, 0x97, + 0x03, 0x54, 0x4c, 0x22, 0x88, 0xc3, 0x82, 0xfd, 0x91, 0xc1, 0xf1, 0x63, + 0xb4, 0x50, 0x46, 0x11, 0x64, 0x07, 0xfd, 0x85, 0xe5, 0x78, 0x57, 0xdd, + 0x19, 0x2a, 0x6b, 0x64, 0x3e, 0xec, 0xb8, 0xf3, 0xb5, 0x95, 0x29, 0x72, + 0xf1, 0x9d, 0xdd, 0xb9, 0xad, 0xd0, 0x78, 0x26, 0x86, 0x10, 0x10, 0x19, + 0xe4, 0x79, 0xae, 0xdc, 0x56, 0xb7, 0x54, 0x4f, 0x94, 0xc6, 0x26, 0x9a, + 0x93, 0xa8, 0x2e, 0x1b, 0x1c, 0xda, 0x87, 0x3a, 0xa2, 0x44, 0xb9, 0x0b, + 0x0f, 0xab, 0x70, 0x3b, 0xb7, 0x6c, 0xbf, 0x58, 0x67, 0x32, 0x7d, 0xa3, + 0x2a, 0xcb, 0x4e, 0x02, 0x92, 0xa1, 0x26, 0x0e, 0x20, 0x5e, 0xb3, 0xec, + 0xc4, 0x04, 0x5b, 0x7f, 0xe5, 0xbd, 0x30, 0xeb, 0xc8, 0xdd, 0xf1, 0x72, + 0x5a, 0x7e, 0xcb, 0x93, 0x22, 0xa0, 0x01, 0x9f, 0xbb, 0x24, 0x9f, 0x50, + 0x01, 0x1f, 0x24, 0x02, 0x85, 0x6d, 0xe6, 0x4d, 0x55, 0xc4, 0x07, 0xe9, + 0x87, 0x38, 0xbf, 0x1a, 0x3b, 0x05, 0x82, 0xc4, 0x73, 0x4b, 0x87, 0x3c, + 0xb4, 0x0a, 0x48, 0x8c, 0x06, 0x67, 0xe7, 0xbf, 0xcc, 0xe7, 0xe5, 0xc3, + 0xb2, 0x81, 0x60, 0xe2, 0xd1, 0xb1, 0x8f, 0x98, 0xbd, 0x7d, 0xbd, 0x4e, + 0x9a, 0xca, 0xbe, 0xcb, 0x81, 0x47, 0x25, 0xaa, 0xfa, 0x91, 0xcf, 0x78, + 0xce, 0xcb, 0x1a, 0x11, 0x79, 0xcf, 0x97, 0xa3, 0x95, 0x95, 0x6f, 0xd7, + 0xae, 0x80, 0xc9, 0xd5, 0x95, 0xb7, 0xcf, 0xe2, 0x9d, 0x98, 0x65, 0x80, + 0xfd, 0x2e, 0xee, 0x46, 0x5e, 0x46, 0x8c, 0xde, 0x52, 0xb4, 0xdc, 0xce, + 0xa8, 0xab, 0x4e, 0x0c, 0x12, 0x9f, 0x89, 0x9c, 0x84, 0x80, 0xfe, 0x08, + 0x64, 0x12, 0x12, 0x95, 0x62, 0xea, 0x65, 0xcc, 0x34, 0x80, 0xcf, 0x92, + 0x5f, 0xc2, 0xae, 0x76, 0xe7, 0x2f, 0xbb, 0xa8, 0xdb, 0x6a, 0x66, 0x60, + 0xaf, 0x88, 0xba, 0x65, 0x32, 0xcf, 0xf7, 0x6e, 0xd8, 0xd0, 0x69, 0xb0, + 0x12, 0x23, 0xd6, 0xc2, 0x32, 0xe5, 0x8e, 0x51, 0xc5, 0x61, 0x28, 0x45, + 0xf7, 0xf9, 0xea, 0x73, 0xce, 0x04, 0x2d, 0x56, 0x43, 0x10, 0x8b, 0x4f, + 0x6b, 0xfa, 0x32, 0xa8, 0x92, 0x8f, 0xd9, 0xb4, 0xfd, 0xa4, 0x74, 0xa8, + 0xea, 0xca, 0xd3, 0x84, 0xbb, 0x5a, 0x34, 0x57, 0xf9, 0xda, 0x25, 0x40, + 0x1f, 0x5e, 0xc2, 0x66, 0x43, 0x05, 0xdd, 0x13, 0x88, 0x91, 0x60, 0xa1, + 0x75, 0xd3, 0xc4, 0x27, 0xff, 0xda, 0x24, 0x3d, 0xd9, 0xd7, 0x47, 0x46, + 0x30, 0xd0, 0x76, 0xc4, 0x9e, 0x97, 0xe3, 0x43, 0xd7, 0x45, 0xaf, 0x49, + 0x36, 0xf2, 0x18, 0xdd, 0x3f, 0x86, 0x9a, 0xec, 0x9a, 0x70, 0xeb, 0x5a, + 0xe2, 0xa0, 0x4b, 0x45, 0x21, 0xb3, 0x32, 0x3d, 0x0c, 0x8c, 0x03, 0x13, + 0xae, 0x46, 0xb5, 0x1a, 0x0a, 0x03, 0x36, 0xfe, 0xfe, 0xfa, 0xc9, 0x4d, + 0x46, 0xf8, 0xfe, 0x6f, 0x99, 0x8c, 0xe4, 0x77, 0x0c, 0x27, 0x59, 0xf7, + 0xc3, 0xfc, 0x32, 0xb3, 0xa5, 0xae, 0xdc, 0x49, 0xac, 0x31, 0x27, 0xa6, + 0x14, 0x92, 0xfb, 0xe3, 0x69, 0x35, 0x8d, 0xa0, 0x50, 0x55, 0x09, 0x90, + 0xdf, 0x67, 0x08, 0x4c, 0x0e, 0xaf, 0x71, 0xc2, 0xe8, 0xb8, 0xdc, 0x45, + 0xe3, 0x6d, 0x58, 0x3f, 0x19, 0x8d, 0xcd, 0xeb, 0xe3, 0x02, 0x49, 0xd8, + 0xc8, 0x8b, 0x29, 0xb3, 0xef, 0x2b, 0xf0, 0x39, 0x5c, 0x11, 0xaa, 0x52, + 0x44, 0x0d, 0x1a, 0x3a, 0x7a, 0x62, 0xda, 0x6d, 0xe3, 0xdd, 0x03, 0x30, + 0x6d, 0x3e, 0x18, 0x30, 0x1d, 0xc0, 0xd0, 0x05, 0x67, 0x98, 0xf5, 0x2a, + 0xc7, 0xa1, 0x58, 0xd7, 0xf8, 0x6f, 0x7d, 0x07, 0x59, 0x27, 0x95, 0xb9, + 0x8d, 0x4d, 0xd7, 0xc8, 0x5e, 0x8b, 0x89, 0x14, 0xb7, 0x1b, 0x35, 0xaa, + 0x72, 0x02, 0x39, 0x3c, 0x41, 0x7c, 0x91, 0x93, 0x81, 0xe1, 0xad, 0xbe, + 0x77, 0x28, 0x80, 0xa2, 0x9c, 0xa8, 0x00, 0x18, 0xa5, 0x70, 0xec, 0xec, + 0x96, 0x95, 0x37, 0xa3, 0xee, 0x15, 0xa0, 0x69, 0x0e, 0x05, 0xb5, 0xb4, + 0xb6, 0xa7, 0x8b, 0xb9, 0x41, 0x88, 0x4f, 0x56, 0x39, 0xa7, 0xbe, 0x24, + 0xce, 0x4c, 0xe0, 0x9c, 0x24, 0x5a, 0xa1, 0xab, 0xcd, 0x82, 0xf1, 0x16, + 0x3f, 0xc0, 0xaf, 0xe1, 0x42, 0xe0, 0x7d, 0x1b, 0xd9, 0x8f, 0xb8, 0x04, + 0xa1, 0x88, 0xd9, 0xc3, 0xaf, 0x4f, 0xda, 0xfd, 0x0b, 0x5c, 0xc3, 0x04, + 0xf3, 0xdb, 0xe6, 0x76, 0x6e, 0xe9, 0xdc, 0xea, 0x6f, 0xa2, 0xa5, 0x75, + 0x2c, 0xc7, 0x91, 0x7d, 0x4b, 0xd5, 0x68, 0x55, 0xbb, 0x2d, 0x14, 0xdb, + 0x06, 0x76, 0xf7, 0xcc, 0x0a, 0x88, 0x6c, 0x2b, 0xa1, 0x57, 0xd6, 0x15, + 0x9c, 0x46, 0xcf, 0x5b, 0x6f, 0x9e, 0x7e, 0xc5, 0x39, 0xda, 0x97, 0x26, + 0x5e, 0xf5, 0x25, 0x06, 0xed, 0x8e, 0x9b, 0x1d, 0x1b, 0x91, 0x07, 0x89, + 0x08, 0xce, 0xd7, 0x38, 0x43, 0x64, 0x8e, 0xf5, 0x3a, 0x52, 0x4a, 0xfb, + 0x3e, 0xff, 0x2c, 0xb3, 0x78, 0x40, 0xb5, 0xdd, 0xb2, 0x8a, 0xd3, 0x6a, + 0xc5, 0xb0, 0xa3, 0x4a, 0xb8, 0xe7, 0x27, 0xa0, 0x5a, 0x8f, 0x0f, 0xda, + 0x53, 0x49, 0xc9, 0x77, 0x2a, 0xef, 0x78, 0xc6, 0xec, 0xaf, 0x10, 0xe5, + 0x71, 0xc5, 0x7a, 0x85, 0xdf, 0xb2, 0x85, 0x02, 0xe3, 0x55, 0x7a, 0x91, + 0x3a, 0x68, 0xb2, 0x9d, 0x3d, 0xd9, 0x01, 0xc5, 0x5f, 0x3c, 0xa8, 0x1d, + 0x99, 0xc6, 0xe7, 0xad, 0x09, 0xd1, 0x39, 0x3a, 0x92, 0xc5, 0x77, 0x9c, + 0xdf, 0x99, 0x56, 0x9f, 0xfe, 0xf8, 0xfd, 0xc8, 0x4f, 0x19, 0xa3, 0xa0, + 0xdf, 0xff, 0x17, 0xac, 0xa9, 0x03, 0x32, 0x85, 0x4c, 0x29, 0xca, 0x89, + 0x58, 0xdc, 0x88, 0xdd, 0xeb, 0x79, 0x68, 0x5e, 0x0f, 0x37, 0x1a, 0xf7, + 0x05, 0xfd, 0x39, 0x91, 0x25, 0x61, 0xf3, 0x04, 0xda, 0x97, 0xfc, 0x7b, + 0xcc, 0x40, 0x63, 0xfd, 0x5b, 0x3b, 0x27, 0x8e, 0x92, 0x6d, 0x98, 0x0f, + 0xcc, 0x9c, 0x9b, 0xda, 0xb2, 0xc6, 0xca, 0x56, 0xff, 0x7e, 0xcc, 0xa2, + 0xc0, 0x45, 0x3e, 0xf6, 0xdf, 0xa7, 0xe8, 0x2a, 0xef, 0x0c, 0xde, 0xec, + 0xa4, 0x1d, 0x2c, 0x3e, 0x03, 0xfd, 0xa4, 0x44, 0x60, 0x4a, 0xf5, 0x83, + 0x8f, 0x09, 0x2d, 0xe8, 0xd5, 0x46, 0xf6, 0x1c, 0x2d, 0x39, 0x28, 0x0c, + 0xdf, 0xa1, 0x2b, 0x05, 0x6e, 0x3c, 0x36, 0xdd, 0x91, 0x81, 0x52, 0xf1, + 0x56, 0xdc, 0xbb, 0x79, 0x62, 0xd8, 0x2e, 0x27, 0x5d, 0x9f, 0x3c, 0xce, + 0x81, 0x5c, 0x70, 0xe5, 0x4d, 0x33, 0x06, 0xd5, 0x14, 0x04, 0xb7, 0xbc, + 0x7b, 0x7a, 0xb4, 0xf7, 0x4a, 0x48, 0x8f, 0x97, 0x85, 0x96, 0x69, 0xc9, + 0x40, 0x52, 0xb1, 0x1c, 0x28, 0x82, 0xb3, 0x63, 0xee, 0x94, 0x2f, 0xcb, + 0x40, 0xad, 0xd7, 0x78, 0xb1, 0xc4, 0x21, 0x05, 0x36, 0xd9, 0x46, 0xf0, + 0x83, 0xcd, 0xee, 0x52, 0x7a, 0xa6, 0xa4, 0x40, 0xb0, 0x2f, 0xf0, 0x1c, + 0xfa, 0x42, 0x98, 0x54, 0x5b, 0xfe, 0x5e, 0xd6, 0x84, 0x73, 0xca, 0x39, + 0xbe, 0x87, 0xf2, 0x92, 0xee, 0x3d, 0x21, 0xcc, 0x69, 0x81, 0xe5, 0xe8, + 0x8a, 0xc3, 0x23, 0x64, 0x98, 0xd5, 0x1d, 0xcd, 0x5c, 0x6c, 0x37, 0xc8, + 0x8b, 0x08, 0x22, 0x12, 0x9f, 0x85, 0xc9, 0xed, 0xb4, 0xa6, 0x07, 0xe1, + 0x62, 0x79, 0x35, 0x5d, 0x26, 0x11, 0x4a, 0x6b, 0x33, 0x37, 0x91, 0x78, + 0xe8, 0xe2, 0xba, 0x8b, 0x8a, 0xb7, 0xbb, 0x0f, 0xd2, 0xb3, 0xa2, 0x02, + 0x0c, 0x57, 0x35, 0x99, 0x88, 0x6b, 0x9b, 0x64, 0x79, 0x1f, 0x4a, 0x48, + 0xd4, 0x3b, 0x5c, 0xeb, 0xb4, 0x83, 0xc3, 0xad, 0x9c, 0x6a, 0xb0, 0xcf, + 0x7f, 0x70, 0xe8, 0x22, 0x46, 0x25, 0xfe, 0x7e, 0x02, 0x44, 0x83, 0x02, + 0xb3, 0x08, 0x2e, 0x34, 0x08, 0x4b, 0xff, 0xa2, 0xc1, 0x60, 0xbb, 0xd8, + 0x89, 0x16, 0xf8, 0xaa, 0xab, 0xea, 0xf7, 0xa0, 0x10, 0x9a, 0xc9, 0xe9, + 0xa4, 0x81, 0xa7, 0x87, 0x32, 0x5b, 0xc1, 0xd0, 0xd9, 0x70, 0x6f, 0xb6, + 0x7c, 0x65, 0xd5, 0x0e, 0x65, 0x93, 0xfe, 0x6d, 0x66, 0xaa, 0xab, 0xd0, + 0x03, 0x07, 0xf2, 0xbe, 0x39, 0xd6, 0xc8, 0xac, 0xf2, 0x06, 0x58, 0x58, + 0x46, 0xc0, 0x1a, 0xbd, 0xa4, 0x96, 0x38, 0x31, 0x32, 0x89, 0x04, 0xdf, + 0xcd, 0x3c, 0x2e, 0x98, 0xb8, 0x39, 0xba, 0xe2, 0xca, 0x6b, 0xd0, 0x53, + 0xce, 0x4a, 0xc8, 0x95, 0x81, 0x84, 0x17, 0xce, 0x7f, 0x1d, 0xc1, 0x5a, + 0xc4, 0xc2, 0x73, 0x30, 0x6d, 0x0b, 0x8c, 0xf8, 0x66, 0x38, 0x4e, 0xa3, + 0x14, 0x84, 0x15, 0x36, 0x9e, 0x0d, 0x56, 0x6b, 0xa6, 0x77, 0x65, 0xa4, + 0x2c, 0x77, 0x00, 0x8b, 0x43, 0x57, 0xc6, 0x25, 0xc5, 0xd0, 0x17, 0x79, + 0x6b, 0x5d, 0xbc, 0xcd, 0xc8, 0x25, 0x8f, 0x20, 0x09, 0xcc, 0xbd, 0x80, + 0x10, 0xdf, 0x35, 0xf6, 0x9c, 0x04, 0x80, 0x23, 0xdc, 0x97, 0xe0, 0xba, + 0x29, 0x48, 0x2e, 0x95, 0x0f, 0xb1, 0x9b, 0xc7, 0xe6, 0x0b, 0x89, 0x16, + 0xe2, 0x81, 0x3b, 0x32, 0x69, 0xc4, 0xde, 0xc6, 0x12, 0x09, 0x47, 0xff, + 0x50, 0xe4, 0x45, 0xb7, 0x35, 0xd2, 0x61, 0x9b, 0x52, 0x6e, 0xbe, 0xaf, + 0xd2, 0xeb, 0x0c, 0x50, 0xf1, 0x57, 0x9f, 0x59, 0xe1, 0xc1, 0x4f, 0x8c, + 0x79, 0x07, 0x05, 0xce, 0x8d, 0x64, 0xb2, 0xf0, 0xd3, 0x4f, 0xe1, 0x7b, + 0xfa, 0x30, 0x0a, 0xc2, 0x5d, 0x0c, 0x47, 0x6c, 0x17, 0x77, 0x1f, 0xe5, + 0xd8, 0x14, 0xfd, 0xc1, 0x01, 0x70, 0x51, 0x60, 0xb2, 0x20, 0xfd, 0x86, + 0xbc, 0x19, 0x5e, 0x01, 0xa6, 0x19, 0x3a, 0x21, 0xa5, 0x0a, 0x1c, 0xd9, + 0xa9, 0x78, 0xbb, 0xc9, 0x01, 0x65, 0xe4, 0xb3, 0x48, 0xb8, 0xe1, 0xe7, + 0xb5, 0xf4, 0x4e, 0xa9, 0xb6, 0xe2, 0x5b, 0xeb, 0xf5, 0x76, 0x06, 0x1a, + 0xd9, 0x08, 0x40, 0xff, 0x72, 0xb2, 0xe3, 0x01, 0x50, 0xb1, 0xad, 0xb3, + 0xa3, 0xf6, 0xef, 0x72, 0x05, 0x0c, 0xf4, 0xce, 0x24, 0x2c, 0x63, 0x89, + 0x63, 0x9e, 0x21, 0xb8, 0xb0, 0xbe, 0xc7, 0x45, 0xae, 0x47, 0x2b, 0x9e, + 0x61, 0x81, 0x4c, 0x76, 0x96, 0x7b, 0x18, 0x37, 0x74, 0xcb, 0x00, 0xef, + 0x38, 0x72, 0x24, 0x0a, 0x63, 0xc1, 0x64, 0xd6, 0x41, 0xc8, 0x6a, 0xf1, + 0xe7, 0x11, 0x20, 0x4b, 0xc2, 0x95, 0x70, 0xb8, 0xf8, 0x8f, 0xd9, 0xae, + 0x8c, 0x12, 0xd8, 0x6f, 0x63, 0x30, 0xca, 0x56, 0x46, 0x11, 0xda, 0x49, + 0x1f, 0x84, 0x3d, 0xae, 0xab, 0x78, 0x29, 0x02, 0x6c, 0x43, 0xa3, 0xef, + 0x9d, 0x97, 0x59, 0x15, 0x53, 0xcd, 0xc7, 0x47, 0x65, 0x30, 0xc7, 0xae, + 0x31, 0x4a, 0x41, 0xb4, 0x66, 0x9c, 0xbb, 0x51, 0x0b, 0xbd, 0xe2, 0x7d, + 0x41, 0x2c, 0xd0, 0x75, 0x57, 0x93, 0xce, 0x2e, 0xeb, 0x31, 0x7f, 0x56, + 0xb2, 0xa4, 0x2b, 0x9f, 0xcc, 0xef, 0x6f, 0xf0, 0x77, 0x19, 0xad, 0x4d, + 0x2e, 0x37, 0x00, 0x75, 0x53, 0xae, 0x22, 0x44, 0x69, 0x1c, 0x8a, 0x90, + 0xf2, 0xcd, 0x0f, 0x6b, 0x37, 0xdb, 0xfd, 0x71, 0x64, 0x80, 0xd8, 0x57, + 0x1b, 0x8f, 0xff, 0x14, 0xd4, 0x5f, 0xe1, 0xd1, 0x0f, 0x06, 0x13, 0x61, + 0x29, 0xa9, 0x80, 0x9d, 0xc7, 0x8a, 0xa0, 0xb5, 0xaa, 0xfc, 0xe0, 0xb4, + 0xb4, 0xf0, 0x31, 0xf0, 0xec, 0x78, 0x03, 0x28, 0xb9, 0xf7, 0xd9, 0xa7, + 0xc8, 0xad, 0x2e, 0x16, 0xb8, 0x18, 0x82, 0x43, 0x66, 0x8b, 0xae, 0xb2, + 0x45, 0x2b, 0x0c, 0x9d, 0x69, 0xbd, 0x1b, 0xc5, 0x20, 0xc6, 0x41, 0xe7, + 0x4f, 0x4b, 0x7b, 0x46, 0x3d, 0x7a, 0x6d, 0x9f, 0x13, 0x2e, 0x0f, 0xf3, + 0x85, 0x3e, 0x5b, 0x12, 0xe5, 0xbf, 0x1b, 0x20, 0xc3, 0x5f, 0x6b, 0xf7, + 0xf7, 0xa3, 0xd7, 0x33, 0xd2, 0xcb, 0x18, 0xa5, 0xa4, 0xa2, 0xd3, 0x59, + 0x91, 0x9a, 0x04, 0xfa, 0x9d, 0xa5, 0x55, 0xad, 0x09, 0x5a, 0x1e, 0x0b, + 0x10, 0xd0, 0x46, 0x18, 0xe4, 0x09, 0xe8, 0x1b, 0x44, 0xd3, 0x78, 0x45, + 0xc0, 0xdf, 0xa2, 0xef, 0xfc, 0x59, 0x8a, 0x1b, 0x22, 0x60, 0xc9, 0x58, + 0x7d, 0x65, 0x45, 0xa9, 0xac, 0xd5, 0xd4, 0xc4, 0x44, 0xd3, 0x08, 0x44, + 0x40, 0x4d, 0x3d, 0x7e, 0x39, 0x81, 0x72, 0x15, 0x49, 0xd7, 0x2c, 0xda, + 0x33, 0xaf, 0xc5, 0xb5, 0x8a, 0x3c, 0xbf, 0x81, 0x88, 0x4f, 0x12, 0xe4, + 0xe8, 0xe6, 0x00, 0xb6, 0xd9, 0xcd, 0xb2, 0x70, 0x08, 0x15, 0x72, 0xf6, + 0x46, 0xc7, 0x98, 0x7c, 0x1d, 0x54, 0xd0, 0x66, 0x2d, 0xa1, 0xd8, 0xda, + 0xb0, 0xe5, 0x9f, 0xa3, 0x2f, 0x2c, 0xfb, 0x34, 0xb3, 0x21, 0x8b, 0x61, + 0xf4, 0xce, 0x60, 0x2b, 0xb5, 0x5e, 0x3d, 0x14, 0x2c, 0xbe, 0x19, 0x9d, + 0x5f, 0x01, 0xe1, 0x21, 0x34, 0x11, 0x6b, 0x10, 0xd4, 0x17, 0x58, 0xb3, + 0x0a, 0x30, 0xe4, 0x17, 0x51, 0x0b, 0xf2, 0xbb, 0xa6, 0xb7, 0x00, 0xa2, + 0xe8, 0xa5, 0xa3, 0x41, 0x1d, 0x65, 0x2d, 0x26, 0x93, 0x26, 0x7d, 0xdc, + 0xad, 0x6f, 0x83, 0xeb, 0x66, 0x55, 0xde, 0x60, 0x21, 0x56, 0x19, 0x4f, + 0x9b, 0x7b, 0x26, 0x4a, 0x80, 0xf5, 0xab, 0x8b, 0xbf, 0xe4, 0xb1, 0xa1, + 0xd6, 0x33, 0x32, 0xbf, 0x86, 0x8c, 0x3c, 0xd0, 0x12, 0x03, 0xd4, 0xb9, + 0x23, 0x54, 0x1b, 0x94, 0x2f, 0xa5, 0x34, 0x4d, 0x59, 0x18, 0x33, 0x8e, + 0x8c, 0xf7, 0x1f, 0xc9, 0x6d, 0x75, 0xfb, 0x2a, 0x22, 0x6c, 0x64, 0xb7, + 0x79, 0xd8, 0x3b, 0xf6, 0x4e, 0x98, 0xd8, 0xa8, 0x2c, 0x06, 0xd1, 0x92, + 0x32, 0x44, 0xec, 0x38, 0x40, 0x3b, 0x53, 0x16, 0x40, 0x8f, 0x92, 0x72, + 0x87, 0xa8, 0xb8, 0xc0, 0x8f, 0x25, 0x4c, 0x4f, 0x24, 0xfc, 0x8d, 0xc6, + 0xa6, 0xeb, 0x2f, 0xdf, 0x2f, 0x0d, 0x2f, 0xd3, 0x6e, 0x70, 0x71, 0xfe, + 0xf0, 0x2e, 0xe9, 0x84, 0xd3, 0xc1, 0xd1, 0x70, 0x4b, 0x8f, 0x7b, 0x60, + 0xb0, 0xb7, 0xe3, 0x79, 0x52, 0x6a, 0x6b, 0x26, 0x03, 0x8f, 0x6a, 0x0f, + 0x8d, 0x85, 0xd7, 0x5f, 0xf7, 0x39, 0x31, 0x0e, 0x26, 0x73, 0x84, 0x3f, + 0x9b, 0x10, 0x6f, 0x29, 0x63, 0x14, 0x36, 0xa2, 0xec, 0x44, 0x7d, 0x84, + 0xc6, 0x4a, 0xec, 0xfe, 0xac, 0xcb, 0xe4, 0xfa, 0xf6, 0x68, 0x83, 0x68, + 0xe0, 0x8f, 0xd3, 0x8a, 0x60, 0x73, 0xf1, 0x5c, 0x71, 0x02, 0x0c, 0xa2, + 0x88, 0x2c, 0xa2, 0x35, 0x35, 0x5c, 0x3f, 0xb1, 0xbe, 0xb3, 0x6b, 0x5c, + 0xe1, 0x78, 0x75, 0x40, 0x20, 0x87, 0x67, 0xca, 0x07, 0x1c, 0x9c, 0x02, + 0xc7, 0xf2, 0x9d, 0x1c, 0xda, 0x1b, 0x86, 0x1b, 0xc6, 0xa6, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x93, 0xca, 0x30, 0xae, + 0xea, 0x26, 0x6a, 0x1b, 0x15, 0x46, 0x0a, 0xe3, 0x57, 0x23, 0x4c, 0x0c, + 0x98, 0x8e, 0x3e, 0xbb, 0x43, 0x14, 0x73, 0xdf, 0x17, 0x91, 0xe2, 0xee, + 0x39, 0xf9, 0xc2, 0x2f, 0xdc, 0xad, 0x0e, 0x00, 0xf5, 0xdd, 0xe3, 0x97, + 0xba, 0x8c, 0xee, 0x53, 0xc4, 0x70, 0x37, 0x46, 0xcf, 0x04, 0xc3, 0xc8, + 0x56, 0x38, 0x2e, 0x39, 0x75, 0x32, 0x6d, 0x98, 0xc4, 0x14, 0xae, 0xa4, + 0x29, 0xa3, 0xc6, 0xb6, 0x66, 0x45, 0x48, 0xdf, 0xc0, 0xa9, 0x4b, 0x4f, + 0xef, 0xb9, 0xb4, 0x89, 0x0d, 0x64, 0x00, 0x5c, 0xd1, 0xc8, 0x2b, 0xf7, + 0xc5, 0x1a, 0x1b, 0x06, 0xb7, 0x49, 0xb1, 0xe3, 0x4d, 0x87, 0xf9, 0x3f, + 0xba, 0x39, 0xa3, 0x56, 0x7f, 0x43, 0xcc, 0x15, 0x9c, 0x3d, 0xba, 0x71, + 0x7b, 0xeb, 0x45, 0x0f, 0x15, 0x1b, 0x6c, 0x84, 0x75, 0x6d, 0x43, 0x0b, + 0x27, 0x12, 0x6b, 0xbc, 0x0a, 0x6d, 0xe4, 0xf6, 0x4f, 0xc7, 0xbb, 0x9e, + 0x91, 0xb5, 0x09, 0x5f, 0x79, 0x2a, 0xbf, 0xda, 0x34, 0x91, 0x44, 0x47, + 0x52, 0x64, 0x00, 0x89, 0x27, 0x17, 0x5c, 0xe9, 0x90, 0x8b, 0xcb, 0xbe, + 0x21, 0x47, 0x65, 0x1c, 0x54, 0x61, 0x48, 0x17, 0x66, 0xb7, 0xa1, 0x60, + 0x27, 0x31, 0x04, 0x42, 0x3b, 0x33, 0x3d, 0xda, 0xf7, 0x61, 0x3d, 0x4b, + 0x91, 0xa5, 0x74, 0x4b, 0xde, 0x16, 0xf2, 0x79, 0x3e, 0xf7, 0x89, 0x87, + 0xb3, 0xdd, 0xa2, 0x49, 0xd7, 0x54, 0x1b, 0x39, 0xff, 0xb5, 0xec, 0x9d, + 0x1d, 0x09, 0x7e, 0x5a, 0x3c, 0xd1, 0xdc, 0x0e, 0x2a, 0x0e, 0x2c, 0x40, + 0x4e, 0xa5, 0x8c, 0x9d, 0xc8, 0x9b, 0xa5, 0xb2, 0x40, 0xa4, 0xaa, 0x3b, + 0xac, 0x93, 0x19, 0xf7, 0xa1, 0x8b, 0xf8, 0x4a, 0x40, 0x08, 0x5d, 0x1d, + 0xb0, 0xae, 0x0f, 0x67, 0xa7, 0x21, 0xaf, 0xe3, 0xb1, 0xfc, 0xff, 0xa0, + 0x95, 0x66, 0x2b, 0xf7, 0x82, 0x2d, 0x8a, 0x26, 0x0f, 0xc3, 0xed, 0x62, + 0xb6, 0xcb, 0x4c, 0x86, 0xe9, 0x20, 0x78, 0x3f, 0x08, 0x53, 0x8f, 0x41, + 0xf1, 0xa1, 0x04, 0x77, 0xd9, 0xe6, 0xea, 0x26, 0x6d, 0x33, 0x48, 0xb3, + 0xbb, 0xed, 0xfc, 0xd7, 0xa3, 0x2b, 0xe2, 0x39, 0xcf, 0x78, 0x4e, 0x11, + 0x26, 0xad, 0x39, 0x83, 0x6e, 0x72, 0xbf, 0xc6, 0x34, 0x23, 0x97, 0x5d, + 0x7b, 0x64, 0x1e, 0x78, 0x00, 0x34, 0x92, 0x5d, 0x3f, 0x23, 0x28, 0x60, + 0x7f, 0x88, 0xf0, 0xca, 0x96, 0x4a, 0x15, 0xbf, 0x8a, 0xb7, 0xd0, 0xd9, + 0x99, 0x8b, 0xdb, 0x26, 0xdc, 0x7e, 0x8d, 0x35, 0x53, 0x60, 0x07, 0x85, + 0x80, 0xc4, 0x9c, 0x0d, 0x81, 0xe2, 0x93, 0x85, 0x76, 0x2d, 0x85, 0x21, + 0x6e, 0xda, 0x29, 0xe5, 0xb1, 0x08, 0x46, 0x09, 0x1b, 0x8a, 0xd9, 0xd2, + 0xd7, 0x16, 0x74, 0xee, 0x26, 0x3e, 0xc4, 0x8c, 0x2e, 0x6b, 0x0c, 0xbc, + 0x95, 0xea, 0x4a, 0xb2, 0xd6, 0x6f, 0x43, 0xd1, 0x3a, 0x8f, 0xbd, 0x77, + 0xb4, 0x67, 0x63, 0x6b, 0xd2, 0xe0, 0xf0, 0x81, 0x74, 0xb7, 0xc5, 0x11, + 0x60, 0x10, 0x6b, 0xc6, 0x0f, 0xfd, 0x84, 0x2e, 0x5c, 0x8f, 0x3b, 0xf5, + 0x68, 0xa7, 0x62, 0xc6, 0x4f, 0xa6, 0xee, 0x19, 0x44, 0xea, 0xc0, 0xe4, + 0x64, 0x12, 0x71, 0x2f, 0xfb, 0xa3, 0x4d, 0xb0, 0x8e, 0x5e, 0xe1, 0x79, + 0x65, 0xd4, 0xf3, 0xed, 0x73, 0x04, 0xf1, 0x6d, 0xc6, 0x75, 0x54, 0x28, + 0x13, 0xe2, 0xd6, 0xa1, 0x26, 0xf9, 0xa4, 0x29, 0x20, 0x5b, 0xd0, 0x3c, + 0x3d, 0xf3, 0x7a, 0x18, 0x9a, 0x3d, 0xec, 0x6a, 0x4c, 0xfd, 0xa5, 0x00, + 0xdf, 0xec, 0xfd, 0x64, 0x38, 0x66, 0xa7, 0xba, 0x59, 0xb3, 0x9b, 0x9c, + 0x44, 0xfb, 0x10, 0x08, 0xb8, 0x79, 0xea, 0x85, 0xbf, 0xa4, 0x14, 0xce, + 0xce, 0x85, 0x22, 0x3f, 0x16, 0x00, 0x1c, 0x57, 0xc8, 0x5a, 0x1b, 0xf5, + 0xff, 0xde, 0x7e, 0xa9, 0xcc, 0xf3, 0xb5, 0x1d, 0x57, 0x06, 0xda, 0xbb, + 0x6c, 0x0a, 0x1e, 0xd4, 0x09, 0x74, 0x84, 0x1d, 0xfa, 0xdf, 0x33, 0x1e, + 0xe2, 0x8f, 0x10, 0xf7, 0x73, 0xab, 0x71, 0xb8, 0x64, 0xce, 0xc0, 0x49, + 0xc0, 0x36, 0xd3, 0x39, 0x31, 0x4c, 0x12, 0x5b, 0xf3, 0xf9, 0xb4, 0x2c, + 0x88, 0xba, 0xd4, 0x1a, 0xbd, 0x0c, 0x99, 0xbd, 0x0e, 0xad, 0x51, 0xe0, + 0xca, 0xdb, 0x25, 0x66, 0x83, 0xe0, 0x55, 0x18, 0xeb, 0xa6, 0x4e, 0x56, + 0xcb, 0x2f, 0xa5, 0xf2, 0x42, 0x7a, 0xa1, 0x05, 0xf0, 0x3a, 0x71, 0x5a, + 0x78, 0x3a, 0x7a, 0x6d, 0x12, 0x9f, 0x43, 0xc5, 0xcc, 0xb3, 0xfd, 0xf2, + 0xbf, 0x05, 0x16, 0xef, 0x07, 0xf9, 0xde, 0x0d, 0x51, 0xf0, 0x33, 0x86, + 0x43, 0x57, 0x40, 0xbc, 0xa9, 0xbd, 0xa0, 0x23, 0xff, 0xbb, 0xe6, 0x15, + 0xa1, 0xeb, 0xe9, 0x78, 0x0d, 0x72, 0x76, 0xf2, 0xb6, 0x6e, 0x46, 0xe2, + 0x86, 0xab, 0x3c, 0x52, 0x2c, 0xc6, 0x77, 0xdd, 0x57, 0xf7, 0x4d, 0x36, + 0xbb, 0x41, 0x08, 0x21, 0xaa, 0xe6, 0x44, 0x50, 0xed, 0xaf, 0x18, 0xb3, + 0xdd, 0x6b, 0x57, 0x46, 0x9e, 0x44, 0x93, 0x20, 0xe0, 0x62, 0x95, 0xcd, + 0xcf, 0xe4, 0x96, 0x92, 0xc3, 0x0d, 0x16, 0xb2, 0xc3, 0xf4, 0x0f, 0x3f, + 0x87, 0x17, 0xb9, 0x7b, 0x60, 0x60, 0xfa, 0xfb, 0x81, 0x5c, 0xb3, 0xb7, + 0x89, 0x73, 0xf7, 0x35, 0xf7, 0x27, 0xf1, 0x0e, 0xa4, 0xa1, 0xba, 0xea, + 0x6a, 0xe3, 0x5c, 0x0f, 0xf7, 0x15, 0xbc, 0x28, 0x57, 0x27, 0x8f, 0xd8, + 0xca, 0x82, 0x19, 0xd0, 0xa3, 0x9d, 0xe5, 0xe0, 0x44, 0xbf, 0x78, 0xa4, + 0x09, 0x69, 0x27, 0xa0, 0x69, 0xb5, 0xd4, 0xbe, 0x00, 0xe6, 0x03, 0x97, + 0xbc, 0x8b, 0xfc, 0x25, 0x70, 0xb3, 0x49, 0x30, 0xe3, 0x24, 0x19, 0x77, + 0xb4, 0x93, 0x46, 0x03, 0xe6, 0x22, 0xaf, 0x76, 0xd2, 0x90, 0x00, 0x05, + 0x46, 0xb8, 0xa4, 0xf5, 0x4c, 0xaa, 0x04, 0x63, 0xa0, 0x57, 0xe0, 0x20, + 0x6e, 0x1a, 0xed, 0x21, 0x86, 0xd0, 0x38, 0x5b, 0xe6, 0xa7, 0xb0, 0xe7, + 0x75, 0xe3, 0x76, 0xb3, 0x15, 0x8b, 0xdc, 0x10, 0x52, 0x15, 0x21, 0x7b, + 0xd0, 0xc4, 0x75, 0x26, 0x1d, 0x6e, 0x0d, 0x4c, 0x08, 0x5b, 0x95, 0x9a, + 0xd0, 0xda, 0xbe, 0x23, 0x98, 0xde, 0x60, 0x2a, 0xe9, 0xa4, 0x92, 0xf0, + 0x92, 0x84, 0xdc, 0x86, 0x60, 0xf5, 0x23, 0x31, 0xf5, 0xe9, 0xd6, 0x00, + 0xc1, 0x78, 0xab, 0x05, 0x94, 0xd3, 0x47, 0x4d, 0x32, 0x0f, 0x82, 0xa0, + 0x99, 0x0b, 0xfe, 0x6b, 0x58, 0xf9, 0x24, 0xf6, 0x17, 0xa0, 0x5f, 0x24, + 0x6a, 0xc6, 0x01, 0xa8, 0xfa, 0xca, 0xdc, 0xb6, 0x83, 0xcb, 0xd2, 0x3b, + 0xb7, 0x0b, 0x04, 0x3e, 0x6a, 0xaf, 0x23, 0x17, 0x3e, 0x14, 0xce, 0x52, + 0x1c, 0xe3, 0x06, 0x66, 0x29, 0x17, 0x6f, 0x7e, 0x66, 0x06, 0xa9, 0x68, + 0x7f, 0xca, 0xad, 0xa8, 0xb7, 0x2d, 0xa4, 0x5d, 0xa6, 0x16, 0xcd, 0xed, + 0xee, 0x14, 0x96, 0xc8, 0x12, 0x69, 0x4e, 0x70, 0x72, 0x2a, 0x75, 0x82, + 0x08, 0x3f, 0x3e, 0x27, 0xa0, 0xea, 0x43, 0x84, 0xa9, 0x9a, 0x91, 0x87, + 0x4f, 0x20, 0x61, 0x55, 0x8d, 0x70, 0xad, 0x6c, 0x59, 0x5d, 0x13, 0x80, + 0xbb, 0x52, 0x55, 0x81, 0x8b, 0x59, 0x94, 0x0f, 0xc2, 0x54, 0x79, 0x59, + 0xe8, 0x9d, 0x58, 0xe5, 0x91, 0x10, 0xb3, 0xef, 0x1c, 0xda, 0xaa, 0xdd, + 0x91, 0x0b, 0xb0, 0x14, 0x3b, 0xad, 0x02, 0x98, 0x40, 0x3c, 0x54, 0xc4, + 0x23, 0xb9, 0x40, 0x54, 0x7e, 0x88, 0x10, 0x3e, 0x24, 0xe5, 0xf6, 0xdf, + 0x5c, 0x9e, 0x7a, 0x9f, 0xd0, 0xff, 0x5e, 0x9c, 0xb6, 0x30, 0x17, 0x94, + 0xd2, 0xaa, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x96, 0xff, 0x2f, 0x01, 0x60, 0x2c, 0x1b, 0xe3, 0xc6, 0xcb, 0xa4, 0x41, + 0xa1, 0x44, 0x13, 0x14, 0xe2, 0x44, 0x77, 0x1c, 0x96, 0xe8, 0xe6, 0x4f, + 0x70, 0x99, 0x3a, 0xef, 0xa1, 0x6f, 0x1f, 0x7f, 0xb9, 0xe9, 0x1e, 0x35, + 0x37, 0x5b, 0x94, 0x90, 0x78, 0xcc, 0x8d, 0xcd, 0x6c, 0x9f, 0xf6, 0x73, + 0xed, 0x23, 0xa2, 0x28, 0x64, 0x58, 0x50, 0x64, 0x05, 0xbc, 0xc9, 0x9b, + 0x5a, 0xec, 0x3f, 0x2b, 0x61, 0xcf, 0xa7, 0x35, 0x56, 0x8c, 0x77, 0x68, + 0xd6, 0xcf, 0x9b, 0xc5, 0x62, 0xee, 0x3a, 0xb2, 0xfe, 0x78, 0xba, 0x02, + 0xe7, 0x26, 0x8a, 0x89, 0x30, 0x19, 0xcc, 0xb0, 0x98, 0xbf, 0x30, 0x2c, + 0xae, 0x13, 0x6c, 0x93, 0x86, 0x19, 0x84, 0x13, 0x01, 0x2f, 0x39, 0x4e, + 0x33, 0xd1, 0x15, 0x99, 0xf7, 0x1e, 0xb8, 0x86, 0xdb, 0xb6, 0xf9, 0x56, + 0x42, 0x0e, 0x4a, 0xb1, 0x5e, 0xf0, 0x9a, 0x06, 0x5e, 0xab, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0xcd, 0xde, 0xad, 0x40, + 0x34, 0xcd, 0x79, 0x0a, 0x29, 0x84, 0x05, 0x3f, 0xb5, 0xbe, 0x49, 0x84, + 0x43, 0xcc, 0xa6, 0xe3, 0xe9, 0xdc, 0x84, 0x14, 0xe7, 0xb3, 0x1b, 0x96, + 0xe8, 0xda, 0x35, 0x15, 0x38, 0xf5, 0xb3, 0xb5, 0x91, 0xc3, 0xc3, 0x94, + 0xc6, 0x79, 0xeb, 0xf5, 0x22, 0x78, 0xf0, 0x0b, 0xda, 0xb0, 0x91, 0xa7, + 0x43, 0x71, 0x8e, 0xa6, 0x52, 0x0f, 0x81, 0x06, 0xc8, 0xdf, 0xb5, 0x1f, + 0x92, 0xb0, 0xfe, 0x93, 0x38, 0x4c, 0xf4, 0x17, 0x66, 0x31, 0xea, 0x08, + 0x72, 0xb9, 0xaa, 0xfd, 0x40, 0x8d, 0xbf, 0x56, 0x19, 0xb1, 0xb5, 0x8e, + 0x4e, 0x4e, 0x73, 0x7f, 0x4b, 0x0c, 0x70, 0x94, 0x7c, 0x9f, 0xfc, 0x23, + 0x35, 0xba, 0xd2, 0x23, 0x88, 0x1d, 0x83, 0x28, 0x45, 0xd7, 0x1b, 0x63, + 0xfb, 0x36, 0x86, 0x06, 0xf3, 0x99, 0x81, 0x6e, 0xd7, 0xf1, 0xd4, 0x53, + 0x6d, 0x30, 0x3c, 0x8d, 0xac, 0xc6, 0x9a, 0xd5, 0xe8, 0x4f, 0x11, 0x58, + 0xba, 0xfd, 0x67, 0x06, 0xe7, 0x1a, 0xb4, 0xa1, 0x45, 0x13, 0xf2, 0x3b, + 0xdc, 0x71, 0xf0, 0xc6, 0x53, 0xfc, 0x8b, 0x2f, 0x14, 0xe4, 0xe0, 0xd6, + 0x8c, 0x96, 0x4c, 0x48, 0xc0, 0x30, 0x6e, 0x00, 0x0f, 0x42, 0xfe, 0xa7, + 0x9d, 0x0f, 0xf2, 0x52, 0x58, 0xf9, 0x35, 0x33, 0x99, 0xda, 0xd5, 0x9d, + 0x61, 0x26, 0x6b, 0x80, 0xff, 0x08, 0x51, 0x54, 0x26, 0xfa, 0x8d, 0xfc, + 0x67, 0x60, 0x93, 0x0e, 0xcd, 0x78, 0x41, 0x5a, 0x31, 0x47, 0x14, 0xb0, + 0x65, 0x89, 0x30, 0xcb, 0x0c, 0xc5, 0xa0, 0x37, 0xa8, 0xe0, 0xcf, 0x24, + 0xa4, 0x2f, 0xad, 0xa7, 0x9c, 0xa2, 0xe8, 0x81, 0x17, 0xbe, 0x2f, 0xd5, + 0xd1, 0xa8, 0xff, 0x9d, 0x5e, 0x7f, 0xd9, 0x6c, 0x56, 0xe6, 0xc4, 0x60, + 0x8d, 0xa5, 0x47, 0x5e, 0x43, 0x1e, 0x34, 0x23, 0xb3, 0x6a, 0xdf, 0x6c, + 0xf8, 0xd1, 0x85, 0x11, 0xaa, 0x74, 0x85, 0x71, 0x27, 0xc5, 0x80, 0x37, + 0x60, 0xb4, 0x2b, 0x53, 0x5a, 0xc4, 0x35, 0xd1, 0xe8, 0x4b, 0x01, 0x58, + 0x1f, 0xdb, 0x73, 0xf3, 0x2c, 0x8b, 0xbb, 0x17, 0x36, 0x76, 0x35, 0x6b, + 0xa0, 0x82, 0x47, 0xf5, 0x16, 0x21, 0x41, 0x43, 0xc9, 0x1f, 0x53, 0xf9, + 0xe9, 0x47, 0xf0, 0x9c, 0x6d, 0xe3, 0x23, 0x59, 0x74, 0xdc, 0x1a, 0x8f, + 0x4e, 0x6c, 0x71, 0x83, 0x7e, 0xd0, 0x2b, 0x50, 0x44, 0x86, 0x5f, 0xbf, + 0x60, 0x92, 0xeb, 0x9a, 0x9b, 0xa2, 0xc9, 0x2b, 0xa8, 0xc4, 0x77, 0x4e, + 0x3f, 0xf8, 0xa6, 0x39, 0x50, 0x5c, 0x7e, 0x2a, 0x70, 0xb0, 0x5d, 0x28, + 0xb2, 0x81, 0xa9, 0xaf, 0x16, 0x5e, 0x27, 0xeb, 0x03, 0x0e, 0x82, 0xad, + 0x28, 0x51, 0x16, 0xd1, 0xf4, 0x58, 0x75, 0x1a, 0xf9, 0x6a, 0xbf, 0x73, + 0xd7, 0x84, 0x07, 0x7f, 0x4c, 0x4e, 0x29, 0x02, 0x9b, 0x60, 0x81, 0x85, + 0xa9, 0xbf, 0xc7, 0xa0, 0x8f, 0x8a, 0xdc, 0xa4, 0xc5, 0x17, 0x51, 0x24, + 0x15, 0x28, 0x9e, 0x5e, 0x78, 0x84, 0x21, 0x02, 0xca, 0x26, 0x61, 0x4e, + 0x95, 0xa6, 0x8d, 0xa6, 0x98, 0x7d, 0x1f, 0x84, 0x19, 0x24, 0x8b, 0x31, + 0x76, 0x89, 0x2a, 0x5f, 0xa9, 0xfb, 0xaa, 0x8a, 0x8c, 0xce, 0xe4, 0x30, + 0xd6, 0xec, 0x5b, 0x39, 0xb7, 0x09, 0x80, 0x23, 0x4c, 0xe1, 0x6e, 0x8f, + 0x7c, 0x10, 0xe8, 0x8a, 0x60, 0x35, 0xd7, 0xa3, 0xe0, 0x5f, 0xcd, 0xfa, + 0x3d, 0x8f, 0xd8, 0x5d, 0xec, 0xc9, 0xc5, 0xa0, 0x73, 0x41, 0x89, 0xe5, + 0x39, 0xf2, 0x42, 0xff, 0x08, 0xa0, 0x12, 0xb7, 0x4a, 0x5e, 0x46, 0x06, + 0x31, 0xbd, 0x88, 0x5e, 0x9e, 0x05, 0x17, 0x51, 0xb3, 0xe7, 0x88, 0x10, + 0x19, 0x32, 0xff, 0x8a, 0x1e, 0xce, 0x66, 0xbc, 0x84, 0x1f, 0xed, 0x52, + 0x52, 0x77, 0xe1, 0x5e, 0xa6, 0x21, 0xe4, 0xad, 0x59, 0xca, 0xa3, 0x77, + 0xea, 0x66, 0x28, 0x15, 0x73, 0x3a, 0xfd, 0xe4, 0x75, 0x46, 0x99, 0x59, + 0x5c, 0x7a, 0x9b, 0x9d, 0x11, 0xb4, 0x76, 0x45, 0x06, 0x45, 0x41, 0x1e, + 0x94, 0xb7, 0xd9, 0xb8, 0xcb, 0xbf, 0x71, 0xec, 0xba, 0x9f, 0x4a, 0x1b, + 0xbc, 0xfd, 0x5c, 0x06, 0x64, 0xfd, 0x31, 0x52, 0xc0, 0xe4, 0xa7, 0x21, + 0x2f, 0x22, 0x92, 0xf0, 0x51, 0x33, 0x92, 0x1d, 0x40, 0x3c, 0x01, 0x81, + 0x3b, 0xa8, 0x2e, 0x4e, 0xb6, 0x60, 0xcd, 0xd4, 0x36, 0x3b, 0x2e, 0x1d, + 0x5e, 0x43, 0xd9, 0x94, 0xf1, 0x51, 0xd3, 0x59, 0x94, 0x6a, 0xd5, 0x5f, + 0x1f, 0xd3, 0xa6, 0x55, 0xda, 0x15, 0xf1, 0x3e, 0x2c, 0x60, 0xb8, 0xc3, + 0xda, 0x0e, 0x56, 0x53, 0xea, 0xcd, 0x39, 0x27, 0x94, 0x86, 0x94, 0xb2, + 0x5b, 0xd8, 0x9a, 0x12, 0x94, 0xb0, 0xb6, 0x77, 0x28, 0xba, 0xde, 0xb6, + 0x60, 0x4d, 0x2b, 0x6e, 0x3d, 0xf6, 0xf1, 0x48, 0xf7, 0x77, 0xa1, 0x49, + 0xe0, 0x9f, 0x1e, 0xc9, 0xe6, 0xcb, 0x95, 0x26, 0x61, 0x5a, 0xc9, 0xed, + 0x49, 0x40, 0x17, 0x57, 0x15, 0xfc, 0x3c, 0xb8, 0x28, 0x79, 0xb8, 0x42, + 0x2a, 0xf9, 0xd4, 0x19, 0xb9, 0x5f, 0x41, 0xc2, 0x25, 0xd7, 0x88, 0x34, + 0xb3, 0x25, 0x4e, 0xca, 0xff, 0x9e, 0x59, 0x9a, 0x33, 0xc8, 0x12, 0xf9, + 0xd5, 0x70, 0xc0, 0x8b, 0x43, 0x13, 0xc4, 0x8d, 0x45, 0x99, 0xaa, 0xd7, + 0xeb, 0xb1, 0xe9, 0xb7, 0x5b, 0xab, 0x48, 0xd1, 0x26, 0x60, 0x8c, 0x13, + 0x55, 0x8a, 0x41, 0xd3, 0x68, 0x58, 0xd4, 0xa6, 0x30, 0x6e, 0x88, 0x3e, + 0x81, 0x6e, 0x61, 0x06, 0x13, 0x66, 0xd5, 0x8e, 0x5d, 0x87, 0x4f, 0xd9, + 0xb1, 0x66, 0xb3, 0xc5, 0x88, 0xa9, 0xc0, 0x73, 0xcb, 0x7f, 0x42, 0xec, + 0x96, 0x64, 0xad, 0x72, 0x85, 0x72, 0xaf, 0xeb, 0xa9, 0xc4, 0x17, 0x86, + 0xab, 0xe7, 0x23, 0xd7, 0x96, 0xf7, 0xb2, 0xb3, 0x51, 0xe1, 0x9a, 0x3b, + 0x0e, 0xaf, 0x89, 0xca, 0x7b, 0xf1, 0x70, 0x7b, 0xc7, 0x82, 0xfc, 0xc7, + 0x6c, 0x37, 0xd9, 0x7b, 0x82, 0x0f, 0x94, 0xcf, 0xd1, 0xa9, 0x33, 0xc2, + 0xa4, 0xab, 0xed, 0xad, 0xee, 0x64, 0x5d, 0x04, 0xf2, 0xcb, 0x8e, 0x99, + 0x22, 0x33, 0x69, 0x85, 0x85, 0xb6, 0x1a, 0x9b, 0x09, 0x18, 0xbe, 0xcd, + 0x63, 0xf6, 0x5d, 0x52, 0xbc, 0x26, 0x99, 0x3e, 0x52, 0xe5, 0x0c, 0xc5, + 0xee, 0xdd, 0xbb, 0x07, 0xbc, 0x38, 0xc1, 0x67, 0x96, 0x8c, 0xe6, 0xe4, + 0x18, 0xfa, 0x07, 0x91, 0x48, 0xef, 0x9c, 0x70, 0x9d, 0x5b, 0x1c, 0x0e, + 0xd5, 0xd3, 0x59, 0xee, 0x44, 0x13, 0xf7, 0x00, 0xa6, 0x20, 0xad, 0x65, + 0x1d, 0xb7, 0x96, 0x2f, 0x79, 0x7b, 0x04, 0xa3, 0x10, 0x90, 0x29, 0x8c, + 0xa3, 0x2e, 0x14, 0x39, 0xd3, 0xe4, 0x6e, 0x46, 0xf7, 0x6e, 0x96, 0x68, + 0xd9, 0xef, 0x45, 0xf7, 0x3c, 0xcd, 0xc7, 0xca, 0x33, 0x64, 0x8e, 0x31, + 0x80, 0x48, 0x7b, 0x7c, 0x81, 0x9a, 0x48, 0xff, 0xd5, 0x0d, 0x74, 0xe7, + 0x77, 0x46, 0x61, 0x9b, 0xde, 0xed, 0x83, 0xe9, 0x4f, 0x92, 0xc1, 0x16, + 0xad, 0x44, 0x40, 0x23, 0xce, 0x04, 0x31, 0xbf, 0xcf, 0xe2, 0x5a, 0x68, + 0x5a, 0xf4, 0x0f, 0xe1, 0x87, 0x79, 0xb0, 0x32, 0x0b, 0x09, 0x6b, 0x72, + 0x2b, 0x16, 0x06, 0x67, 0x82, 0x0b, 0x92, 0x35, 0xdb, 0x4c, 0xe2, 0x4a, + 0x60, 0x99, 0xaf, 0x52, 0x10, 0x4b, 0xa5, 0xcf, 0xac, 0x66, 0x49, 0x56, + 0x04, 0xc0, 0xd6, 0x6f, 0x62, 0x53, 0x6f, 0xcb, 0x62, 0xe9, 0xa5, 0xca, + 0x18, 0x8e, 0x86, 0x3f, 0x36, 0xfd, 0xea, 0x55, 0x16, 0x6d, 0x6c, 0x6a, + 0x8f, 0xa7, 0x9c, 0x70, 0x15, 0xd7, 0xf4, 0x57, 0x68, 0x04, 0x84, 0x60, + 0x3b, 0xb0, 0x32, 0xc4, 0xea, 0x9d, 0x70, 0xb9, 0xa6, 0x34, 0xe5, 0xfa, + 0xa1, 0x24, 0x54, 0x7f, 0xef, 0xac, 0xb4, 0x5f, 0xa0, 0xc0, 0x40, 0x3f, + 0x73, 0xdf, 0x56, 0xa6, 0xd9, 0x17, 0xf4, 0xff, 0x50, 0xae, 0x21, 0x0d, + 0x5a, 0xe0, 0xb0, 0xf9, 0x5b, 0x7a, 0x61, 0x6e, 0xa6, 0x85, 0x85, 0xbf, + 0x19, 0x03, 0xe2, 0x74, 0x1f, 0x03, 0x70, 0x76, 0x3c, 0xed, 0x02, 0x7d, + 0xfa, 0xf9, 0x1e, 0x17, 0xdd, 0x42, 0x30, 0xf0, 0x32, 0x47, 0x46, 0xae, + 0xf5, 0x64, 0xe6, 0x5e, 0x2b, 0x40, 0x86, 0x97, 0xb1, 0x24, 0x52, 0x69, + 0x67, 0x79, 0x8e, 0x0d, 0xcc, 0x07, 0xcb, 0x72, 0x29, 0xe9, 0xba, 0x2d, + 0xf7, 0xcb, 0xe3, 0x86, 0x06, 0xaa, 0x6d, 0x79, 0xf8, 0xb6, 0x93, 0x0a, + 0x9c, 0x97, 0xef, 0x47, 0x37, 0x13, 0x2e, 0x6b, 0xfd, 0x59, 0x0c, 0xc9, + 0x5e, 0x5e, 0xcd, 0x71, 0x6f, 0x99, 0x0d, 0x88, 0x9d, 0xbb, 0x7c, 0x2b, + 0x22, 0xd5, 0xbe, 0xee, 0x26, 0x1c, 0xe1, 0xad, 0xc8, 0x4d, 0x5f, 0x6b, + 0xd1, 0xf4, 0x30, 0x4d, 0x46, 0x1d, 0x54, 0x11, 0x4b, 0xa0, 0x7f, 0x94, + 0x71, 0xc0, 0x44, 0x4a, 0x42, 0x11, 0xf5, 0x89, 0xec, 0xb5, 0x24, 0x45, + 0xf1, 0xf0, 0x30, 0x54, 0xf8, 0x62, 0xdb, 0x58, 0x3d, 0x7c, 0x2a, 0x82, + 0xe5, 0xbe, 0x13, 0xcf, 0xdc, 0x88, 0xfb, 0xd3, 0x1e, 0x4d, 0xa5, 0x3e, + 0xad, 0x95, 0xa2, 0xe6, 0x48, 0x73, 0xb2, 0xbe, 0x96, 0xef, 0x8e, 0x0b, + 0x28, 0xf9, 0xbe, 0x2a, 0xd6, 0x68, 0x9e, 0x9c, 0x7b, 0x5a, 0xaf, 0x20, + 0xf6, 0xa5, 0x3f, 0x99, 0x61, 0x57, 0xe8, 0x1c, 0xb2, 0xc3, 0xd0, 0x7f, + 0x2c, 0xb5, 0xe9, 0x66, 0x8e, 0x88, 0xec, 0x13, 0x51, 0xbc, 0x8e, 0xb6, + 0xe2, 0x91, 0xbf, 0x5e, 0x8c, 0x1c, 0xdd, 0x0e, 0x0a, 0x13, 0x06, 0xc6, + 0x62, 0x1c, 0x41, 0x8d, 0xa1, 0xc0, 0xf2, 0xfa, 0x76, 0x35, 0xaa, 0x77, + 0x06, 0x3f, 0x76, 0x50, 0xf6, 0x43, 0xf2, 0x25, 0x00, 0x79, 0xde, 0xca, + 0xa1, 0x06, 0x6f, 0xb4, 0x17, 0x4b, 0x99, 0x5a, 0x00, 0x32, 0xd6, 0xb0, + 0x1f, 0x80, 0x53, 0x16, 0xaa, 0x87, 0x72, 0xa2, 0x34, 0xaf, 0x90, 0x3d, + 0x60, 0xde, 0x0e, 0x6d, 0x83, 0xda, 0xb2, 0x11, 0x2f, 0x39, 0xdc, 0x1a, + 0xfe, 0x51, 0x74, 0x10, 0x3c, 0x41, 0xd5, 0x41, 0x65, 0x4a, 0xa0, 0x11, + 0xde, 0x95, 0x34, 0xef, 0xa0, 0xc9, 0xa8, 0xd3, 0xcb, 0xb9, 0x7d, 0x51, + 0x7d, 0xff, 0x26, 0x88, 0xd8, 0x29, 0x0e, 0xa0, 0xd4, 0xa7, 0x07, 0x33, + 0xe7, 0x7d, 0x59, 0x9f, 0x35, 0xc1, 0xb5, 0xf7, 0x78, 0x78, 0x84, 0xf0, + 0x20, 0x41, 0x3f, 0x02, 0x7d, 0x41, 0x90, 0x01, 0x8d, 0xa4, 0xd8, 0xd7, + 0xeb, 0x56, 0x7f, 0x38, 0xbc, 0x1e, 0x15, 0xdf, 0xfc, 0x34, 0xe7, 0x99, + 0xd4, 0x92, 0xd5, 0xf3, 0x9e, 0x16, 0x0b, 0x5c, 0xeb, 0xb6, 0x78, 0xac, + 0x84, 0x06, 0x8e, 0xfe, 0xd0, 0x7c, 0xce, 0x4a, 0x43, 0x49, 0x3b, 0xe1, + 0xab, 0x57, 0xc0, 0x12, 0xd6, 0x9d, 0xa4, 0xee, 0x91, 0x10, 0x81, 0xe2, + 0xfc, 0x02, 0x26, 0x7a, 0xca, 0x81, 0x5b, 0x2f, 0x34, 0x51, 0xdd, 0x25, + 0x4d, 0xc8, 0xf9, 0x3e, 0x59, 0x0f, 0x3d, 0x64, 0x51, 0xbf, 0x42, 0xc4, + 0x92, 0x9d, 0x8f, 0x39, 0x8a, 0x31, 0x09, 0x24, 0x19, 0x44, 0xc0, 0xf4, + 0xea, 0xca, 0x59, 0xcb, 0x86, 0x6c, 0x02, 0x7a, 0xe5, 0x30, 0x79, 0xe2, + 0x2c, 0x76, 0x08, 0x8f, 0x98, 0x0d, 0x4d, 0x12, 0xc3, 0x98, 0xb4, 0x24, + 0x04, 0x4f, 0x51, 0xec, 0x4e, 0xec, 0xbd, 0x8c, 0xc4, 0x79, 0x51, 0x7f, + 0xe1, 0xce, 0x76, 0x28, 0x0b, 0x7b, 0xc5, 0x3f, 0x5b, 0x48, 0x19, 0x76, + 0x68, 0x31, 0x8e, 0x28, 0xff, 0x18, 0x24, 0xe3, 0x91, 0xe7, 0x49, 0x0d, + 0x10, 0xbd, 0x00, 0xc6, 0x58, 0xfd, 0xb6, 0x88, 0x63, 0xbd, 0xb4, 0x4b, + 0xb8, 0xed, 0xdd, 0xb7, 0x53, 0xce, 0x89, 0xdb, 0x7f, 0xf4, 0xc3, 0x21, + 0x31, 0xad, 0x20, 0x78, 0x06, 0x71, 0xaf, 0xc0, 0xe3, 0xdc, 0xb8, 0xf4, + 0x80, 0xc8, 0x33, 0x1d, 0x8b, 0xff, 0x5a, 0x92, 0x68, 0x4d, 0xc1, 0x5b, + 0x58, 0x3e, 0xf6, 0x7f, 0xba, 0x42, 0xa5, 0x6d, 0xec, 0x03, 0x36, 0xc9, + 0x3f, 0x83, 0x1f, 0x0c, 0x33, 0x57, 0x6a, 0x43, 0x5f, 0x11, 0x72, 0x19, + 0x2c, 0xda, 0x71, 0x58, 0xf2, 0x50, 0x50, 0x06, 0x97, 0xd0, 0xdf, 0xd1, + 0x4f, 0x0b, 0x00, 0x1a, 0xea, 0x85, 0x3b, 0x37, 0x2f, 0xf0, 0x40, 0x52, + 0xd9, 0x2a, 0xe8, 0x54, 0xa5, 0xee, 0x0f, 0x49, 0x74, 0x39, 0x96, 0x5d, + 0x60, 0x8f, 0x14, 0x59, 0x86, 0x59, 0x86, 0xfb, 0x67, 0x71, 0x5c, 0x26, + 0x5f, 0xe9, 0xab, 0x32, 0x77, 0x83, 0xdf, 0x02, 0x19, 0x85, 0xae, 0x4d, + 0x7d, 0x9c, 0x8d, 0x4f, 0x61, 0x05, 0x3c, 0x0c, 0xc6, 0x74, 0x9e, 0x36, + 0x33, 0xb8, 0x14, 0x85, 0xab, 0xa2, 0x0b, 0x5d, 0x22, 0xf2, 0x50, 0x3e, + 0xa4, 0x88, 0xac, 0x67, 0xf9, 0x06, 0xe5, 0x30, 0x8e, 0xf9, 0x67, 0x34, + 0xd5, 0x94, 0x5b, 0x35, 0xb7, 0x3d, 0x39, 0x5f, 0x4e, 0xae, 0xfe, 0xf7, + 0x57, 0xd3, 0x95, 0x7b, 0x0a, 0xd9, 0x92, 0x4a, 0x66, 0x29, 0xa0, 0x18, + 0x35, 0x54, 0x14, 0x44, 0x79, 0x72, 0xc3, 0xbc, 0xa8, 0x1a, 0xd3, 0xa3, + 0xbe, 0x6f, 0x9e, 0xcc, 0x68, 0xb6, 0x5f, 0xd4, 0x42, 0xab, 0xe8, 0x09, + 0x60, 0x57, 0x2e, 0xb2, 0x9a, 0x5b, 0x62, 0x38, 0xfb, 0x0a, 0x35, 0x9c, + 0x4f, 0xf7, 0xe0, 0xd2, 0x06, 0x04, 0x1f, 0x79, 0x7f, 0xa7, 0x7b, 0xd3, + 0x63, 0xc9, 0xbd, 0x16, 0x58, 0x38, 0x7b, 0xaa, 0x08, 0xf3, 0x14, 0x6c, + 0x25, 0xf8, 0xa5, 0xe9, 0x4b, 0x45, 0x34, 0x89, 0x76, 0x74, 0xcb, 0x41, + 0x9c, 0x2a, 0xd9, 0xca, 0xb3, 0x12, 0x46, 0x6d, 0x85, 0x4d, 0x63, 0x2d, + 0x24, 0x1b, 0x19, 0x6b, 0x3f, 0x61, 0x6b, 0x4b, 0x15, 0x83, 0x2d, 0x8f, + 0x61, 0xab, 0xd1, 0x55, 0x93, 0x4e, 0x26, 0xd6, 0x7a, 0x0a, 0x8a, 0xff, + 0x58, 0x44, 0xf7, 0x39, 0x31, 0x1a, 0xab, 0xa6, 0x98, 0x31, 0x41, 0x03, + 0xb6, 0xc9, 0xf5, 0x50, 0xe3, 0x7b, 0xc0, 0x59, 0x74, 0x60, 0x91, 0xb4, + 0x79, 0x02, 0x25, 0xc1, 0xb5, 0xbd, 0xcb, 0x6e, 0x40, 0x61, 0xfe, 0x68, + 0x29, 0x83, 0x1b, 0xd2, 0x49, 0xe1, 0x31, 0xde, 0xdd, 0x53, 0xb0, 0xb8, + 0x96, 0xa2, 0xce, 0xea, 0x8b, 0x66, 0x2c, 0x5a, 0x80, 0x51, 0x0b, 0xc1, + 0x2d, 0x9a, 0xfa, 0x9d, 0xc6, 0xcc, 0x2b, 0xbb, 0xaa, 0xce, 0x98, 0xaa, + 0x26, 0x15, 0x8f, 0x4a, 0xe7, 0xdb, 0x17, 0x6c, 0xe5, 0x58, 0xc9, 0xae, + 0xe4, 0x9c, 0x1d, 0xab, 0x59, 0x84, 0x3e, 0x27, 0x76, 0x03, 0xe3, 0x82, + 0x64, 0x6f, 0x6e, 0x6f, 0x63, 0xd2, 0x12, 0x84, 0xe3, 0x9b, 0x9d, 0x7e, + 0x53, 0x1a, 0x54, 0x8d, 0xc1, 0xf0, 0x94, 0xae, 0xad, 0x8f, 0x6a, 0x12, + 0x4e, 0xa7, 0x30, 0xdb, 0x55, 0xbe, 0x09, 0xe2, 0x56, 0x08, 0xc4, 0x3a, + 0xb0, 0x55, 0xb0, 0x24, 0x96, 0xa6, 0x3e, 0x28, 0xd0, 0x35, 0xfb, 0x58, + 0x47, 0xba, 0x2d, 0x51, 0xbb, 0x72, 0x20, 0x59, 0xd2, 0xdd, 0x9c, 0xe2, + 0xb5, 0x31, 0x90, 0xac, 0x74, 0x5d, 0x9f, 0x3d, 0x8c, 0x1c, 0x96, 0xc0, + 0x60, 0x61, 0xa8, 0xbb, 0x3c, 0xb3, 0x6d, 0x6d, 0x92, 0x4a, 0xca, 0xbb, + 0x60, 0x5e, 0x82, 0x0d, 0x7f, 0xab, 0x4b, 0x36, 0x4c, 0x93, 0x0d, 0x88, + 0x71, 0xaf, 0xb6, 0x53, 0xb0, 0x38, 0xb4, 0x1c, 0xb4, 0x7b, 0xd4, 0x13, + 0x32, 0x6c, 0xe4, 0xee, 0x6a, 0xb3, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x00, 0x88, 0x83, 0x91, 0x4c, 0x2e, 0x1e, 0xbe, 0xa4, + 0xb5, 0x96, 0xff, 0x67, 0x50, 0xe9, 0x81, 0x0e, 0x5d, 0x0e, 0xad, 0xc4, + 0x1f, 0xeb, 0x98, 0x38, 0xcc, 0x54, 0x9d, 0x27, 0xa6, 0xf1, 0x37, 0x23, + 0xce, 0xb4, 0x5b, 0xff, 0x12, 0xb1, 0xb8, 0x35, 0x5e, 0x03, 0x02, 0x04, + 0xad, 0xa6, 0x6f, 0x43, 0xfc, 0xe4, 0xbe, 0x0c, 0xe0, 0x93, 0xd5, 0xef, + 0x09, 0xfa, 0x04, 0xe9, 0x5a, 0x22, 0xd4, 0x81, 0xc1, 0x27, 0x4f, 0x5f, + 0x6e, 0x83, 0x5a, 0x8a, 0x2d, 0xbb, 0x8f, 0xa4, 0x91, 0xcc, 0x82, 0x37, + 0x3b, 0x14, 0x98, 0x58, 0x86, 0x44, 0xb7, 0xa9, 0x58, 0xf3, 0x3d, 0x49, + 0x71, 0x7a, 0x37, 0xcd, 0xc5, 0xb9, 0xc9, 0x46, 0xd5, 0xd4, 0x17, 0x60, + 0x1a, 0xbf, 0x93, 0xa9, 0xe9, 0x08, 0x25, 0x40, 0xd1, 0x65, 0xae, 0xdd, + 0x85, 0xa6, 0xcc, 0x06, 0xca, 0x91, 0xe1, 0x63, 0xf9, 0x6b, 0x15, 0xa8, + 0x04, 0x61, 0xd2, 0xa6, 0x59, 0x21, 0x1a, 0x1c, 0xc9, 0xa9, 0xa9, 0xc8, + 0x54, 0x86, 0xac, 0xa5, 0xd6, 0x95, 0x39, 0x83, 0x4b, 0x6b, 0x69, 0xa6, + 0x94, 0xd8, 0xc0, 0xfb, 0x66, 0x0f, 0x3a, 0xbe, 0xc7, 0xf3, 0xcc, 0xd5, + 0xb7, 0x1b, 0x60, 0x02, 0x95, 0x45, 0x4a, 0x12, 0xc9, 0xfe, 0x75, 0x7c, + 0x1b, 0xb2, 0x86, 0x96, 0x28, 0x07, 0xa2, 0x18, 0x7a, 0x6c, 0x90, 0x6f, + 0x32, 0x0c, 0xc8, 0x34, 0xbc, 0x75, 0x4d, 0x96, 0x03, 0xa6, 0x0f, 0x3d, + 0x35, 0x1b, 0x64, 0x76, 0x95, 0x55, 0xff, 0x25, 0xd4, 0x71, 0xcf, 0x8a, + 0x73, 0x6d, 0x9b, 0x74, 0xfe, 0xff, 0x9e, 0x31, 0x9e, 0x5e, 0x89, 0x5a, + 0x1a, 0xeb, 0x8d, 0x06, 0x3b, 0xf2, 0xf6, 0x06, 0x5d, 0xc3, 0xba, 0x04, + 0xca, 0x0f, 0x07, 0x2c, 0xbd, 0x54, 0x52, 0xd9, 0x1c, 0x2f, 0x0e, 0x13, + 0x5e, 0x25, 0x13, 0xe5, 0xd7, 0x8e, 0x19, 0x42, 0x1b, 0x52, 0x2e, 0xd2, + 0x8f, 0xc5, 0x8e, 0x1c, 0x34, 0x2e, 0x4d, 0xd5, 0x51, 0x7d, 0x91, 0x64, + 0xbc, 0xb4, 0x0d, 0xc9, 0xe7, 0x1c, 0x6c, 0x47, 0xe9, 0xbb, 0x67, 0x9a, + 0x96, 0xde, 0xad, 0xff, 0xba, 0x35, 0x25, 0x6d, 0x57, 0xa1, 0x93, 0xfe, + 0xe2, 0x8d, 0x02, 0xeb, 0xf0, 0x2f, 0x54, 0xfd, 0x46, 0xc0, 0x8f, 0xea, + 0x32, 0x7b, 0x57, 0xda, 0xe0, 0x29, 0x1c, 0x19, 0xba, 0xa4, 0xa6, 0x1c, + 0x6e, 0xeb, 0x7a, 0xa8, 0x8a, 0xe1, 0xc6, 0x12, 0xf5, 0xa3, 0x24, 0x1a, + 0x96, 0xe1, 0x02, 0xc0, 0xf4, 0x7d, 0x14, 0x72, 0xd6, 0x12, 0x8e, 0x6c, + 0x8c, 0xd2, 0xfd, 0x88, 0x78, 0x48, 0xf3, 0x74, 0x38, 0x86, 0x04, 0x68, + 0x6d, 0x7c, 0xf4, 0x4c, 0x40, 0x17, 0xf6, 0x8f, 0xb2, 0x6c, 0xd7, 0x66, + 0x66, 0x3b, 0x38, 0xa1, 0xbb, 0x1e, 0xff, 0x72, 0x1f, 0x64, 0x56, 0xc2, + 0x53, 0x1c, 0x6f, 0x84, 0x2b, 0xbd, 0x23, 0xd9, 0xb4, 0x6b, 0x87, 0x79, + 0x99, 0xec, 0x81, 0x8d, 0x1a, 0x58, 0x00, 0xf0, 0x2c, 0xc1, 0xc4, 0x57, + 0x74, 0x0f, 0xce, 0x32, 0xe2, 0x5e, 0xae, 0x02, 0x1c, 0xe8, 0x94, 0xc6, + 0x44, 0xaa, 0x7b, 0x9a, 0x32, 0xb5, 0x33, 0xac, 0xfc, 0x41, 0x65, 0xf2, + 0xca, 0xcc, 0xc6, 0x74, 0x36, 0xb2, 0xc9, 0x0e, 0x26, 0x73, 0xae, 0x68, + 0x98, 0xa4, 0x36, 0xe8, 0x98, 0x39, 0xad, 0x05, 0x3f, 0xca, 0x12, 0xcc, + 0x86, 0xfd, 0xc6, 0x57, 0xf0, 0x02, 0x4e, 0x45, 0xcb, 0x54, 0x34, 0xdd, + 0x66, 0x26, 0xab, 0xda, 0x95, 0xa5, 0x85, 0xec, 0x02, 0x03, 0xb6, 0x29, + 0x30, 0x11, 0x40, 0x54, 0x9a, 0x6a, 0x87, 0x2e, 0x97, 0xa1, 0x7e, 0xeb, + 0x34, 0x39, 0x78, 0x3b, 0xbc, 0x5f, 0x8e, 0xc5, 0x0e, 0x21, 0x29, 0x4b, + 0xb7, 0x1b, 0xe7, 0x14, 0x08, 0x34, 0xb7, 0x9a, 0x0a, 0xb2, 0x6c, 0x25, + 0x76, 0xb5, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0xe2, 0x7d, 0x48, 0xdd, 0x1a, 0xcb, 0xb6, 0x5c, 0x6f, 0xbe, 0x32, 0x9d, + 0xd2, 0x2b, 0x9e, 0x10, 0x65, 0xd7, 0x1e, 0xec, 0xc8, 0xb5, 0x10, 0x64, + 0x8f, 0x5d, 0xef, 0xfe, 0x9b, 0x6c, 0x9b, 0x02, 0x6a, 0x6d, 0xf7, 0x98, + 0x7b, 0xf7, 0x17, 0xfd, 0x49, 0x1b, 0x6a, 0xc5, 0x3c, 0xa0, 0xfc, 0xa8, + 0x94, 0x95, 0xed, 0x48, 0x81, 0x04, 0x53, 0x8c, 0xbe, 0xe4, 0x4e, 0xaf, + 0xc1, 0x9d, 0xc3, 0xdf, 0xc2, 0xb5, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0xae, 0xb0, 0x67, 0x5b, 0x99, 0x26, 0x07, 0xfb, + 0x6c, 0x98, 0xfe, 0xbb, 0x35, 0xf1, 0x5b, 0x02, 0xc6, 0x03, 0xfc, 0x97, + 0x21, 0x16, 0x8d, 0x48, 0xd4, 0x4f, 0x03, 0xd9, 0x7c, 0x9f, 0xa6, 0x1e, + 0x6f, 0x5a, 0x58, 0x17, 0x6d, 0x26, 0xb4, 0xc5, 0x4c, 0xe5, 0x93, 0x0a, + 0x9c, 0xb2, 0x40, 0xbc, 0x60, 0xc7, 0x2b, 0xdb, 0x3b, 0xc0, 0x3c, 0x5c, + 0x44, 0x4b, 0xdd, 0x58, 0xbe, 0xdc, 0xc5, 0xb5, 0x6a, 0xf9, 0x5e, 0x73, + 0x07, 0x58, 0x8f, 0x45, 0x7b, 0xac, 0xba, 0x82, 0x96, 0x49, 0x4d, 0x22, + 0x70, 0x7a, 0x3d, 0x69, 0x26, 0x8b, 0x88, 0x13, 0xf1, 0x8d, 0xfc, 0xdf, + 0x73, 0xd5, 0x20, 0x3c, 0x52, 0x92, 0x16, 0xb1, 0x6e, 0xb7, 0x41, 0xbe, + 0x23, 0x9b, 0x51, 0xf7, 0xc9, 0x38, 0x8a, 0xc7, 0x6e, 0x68, 0x82, 0xd1, + 0x59, 0x50, 0x09, 0x4b, 0x44, 0x3b, 0x28, 0x06, 0x60, 0x75, 0x7a, 0xe5, + 0xa1, 0x36, 0xbb, 0x62, 0x44, 0xe3, 0xd0, 0x68, 0x14, 0xea, 0xad, 0xf9, + 0x18, 0xcc, 0xd5, 0x42, 0x5d, 0x18, 0x53, 0xe6, 0x4a, 0xfe, 0xde, 0x32, + 0xe1, 0xe7, 0xf8, 0x8c, 0x9d, 0x35, 0xf4, 0x4a, 0xcb, 0x23, 0x2f, 0x91, + 0xb5, 0xb0, 0xb2, 0x01, 0x5c, 0x22, 0x8c, 0x42, 0x42, 0xd5, 0xf0, 0x82, + 0x6f, 0x9f, 0x64, 0xe5, 0x99, 0x4d, 0x36, 0x0b, 0xfc, 0x78, 0x38, 0x30, + 0x47, 0x8f, 0x0b, 0x57, 0x86, 0x4f, 0x1b, 0xc9, 0x05, 0x0e, 0x08, 0xc4, + 0xf4, 0xab, 0x9e, 0x90, 0xb4, 0x4f, 0x36, 0x54, 0xe8, 0xa1, 0x3f, 0x90, + 0xd2, 0xf3, 0xb4, 0xb4, 0xdd, 0xf3, 0x43, 0x2f, 0xc4, 0x43, 0xbb, 0x99, + 0x8e, 0xb8, 0x61, 0x59, 0x5e, 0xfa, 0x1b, 0x3c, 0xc1, 0xeb, 0x9d, 0x35, + 0x62, 0x34, 0x82, 0x45, 0xef, 0x41, 0xe9, 0xfc, 0x35, 0xae, 0xb4, 0x0b, + 0xce, 0x52, 0x5b, 0x40, 0x7d, 0xdd, 0x86, 0x83, 0x52, 0x74, 0x77, 0x11, + 0xc2, 0x9b, 0x8c, 0xa3, 0x63, 0xc2, 0x2d, 0xdd, 0x8c, 0x76, 0x13, 0xc5, + 0xc0, 0xde, 0x3e, 0x6b, 0xe1, 0x0f, 0xeb, 0x0f, 0x0a, 0x25, 0x41, 0x2f, + 0x8b, 0x4a, 0x98, 0x30, 0xcb, 0x1a, 0x43, 0xa3, 0xc1, 0xcc, 0x44, 0x9a, + 0x6c, 0xdc, 0x92, 0x40, 0xc4, 0x7a, 0x1f, 0x8a, 0x6f, 0x74, 0xf3, 0xf5, + 0x52, 0x72, 0xf7, 0x81, 0x6e, 0x74, 0x75, 0xe6, 0xea, 0xd9, 0x57, 0x91, + 0xae, 0xf2, 0x3f, 0x35, 0x4b, 0x99, 0xd9, 0x3f, 0x85, 0xe0, 0x92, 0xaa, + 0x35, 0xac, 0x28, 0xbf, 0x43, 0xb8, 0xad, 0xc7, 0xc5, 0xf6, 0x15, 0x2f, + 0x7c, 0xfb, 0x34, 0x48, 0xf3, 0x04, 0x12, 0xf4, 0x2f, 0x92, 0x74, 0xc8, + 0xea, 0xbc, 0x24, 0x6e, 0x3b, 0x0e, 0x9e, 0xf0, 0xaf, 0x02, 0x97, 0x95, + 0xbc, 0x90, 0x7f, 0xc4, 0xf8, 0xe2, 0x04, 0x9a, 0x8f, 0xfc, 0xbc, 0x50, + 0xfe, 0xf7, 0x89, 0x17, 0x2c, 0xdb, 0xd6, 0x5e, 0xbf, 0xd9, 0x8e, 0x89, + 0x8b, 0x06, 0x1d, 0x0b, 0x81, 0x2a, 0x55, 0x5c, 0x5f, 0xb6, 0xa6, 0xa5, + 0xd2, 0xaa, 0x79, 0x9c, 0x39, 0x31, 0x76, 0x03, 0x98, 0x42, 0xd6, 0xb7, + 0x37, 0x1f, 0xc8, 0x51, 0x8a, 0x1c, 0x5d, 0xcd, 0x9c, 0x78, 0xa4, 0x22, + 0x6e, 0x12, 0x10, 0x0a, 0x33, 0xc9, 0xe0, 0xfe, 0xfc, 0xe8, 0x15, 0xe7, + 0xef, 0xd8, 0x6d, 0xc7, 0xc9, 0xc2, 0x8e, 0x18, 0x82, 0x2f, 0xa6, 0x09, + 0x8a, 0xdc, 0x41, 0x6b, 0x89, 0xea, 0xd9, 0xd6, 0x96, 0xfd, 0xba, 0x6e, + 0xae, 0x2d, 0x0c, 0xf9, 0x3c, 0x4c, 0x1a, 0xfa, 0x98, 0x83, 0x51, 0x45, + 0x9d, 0x1e, 0xa5, 0xc1, 0x81, 0x54, 0x37, 0x5d, 0x28, 0xca, 0xa6, 0xfe, + 0x48, 0xf4, 0x77, 0x17, 0x92, 0x1d, 0x0c, 0xb3, 0x39, 0x77, 0x22, 0xd9, + 0xc7, 0xc2, 0xaf, 0x70, 0x0a, 0xd3, 0xa6, 0x57, 0x69, 0xfb, 0xb9, 0xe0, + 0xc4, 0x73, 0x7a, 0x68, 0xee, 0x27, 0x6e, 0x3a, 0x6e, 0xae, 0x32, 0xf6, + 0x09, 0xb3, 0x0b, 0x40, 0x72, 0xc6, 0x26, 0x6e, 0xc5, 0x88, 0x6b, 0xce, + 0x99, 0x88, 0x60, 0x6f, 0x6e, 0xa9, 0xe6, 0xd7, 0x35, 0x5e, 0x3b, 0x36, + 0x0d, 0x14, 0xb8, 0x2f, 0xde, 0x67, 0xc8, 0x2e, 0x52, 0xc1, 0xf1, 0x58, + 0x87, 0x32, 0x2a, 0x52, 0x21, 0x27, 0x1e, 0x04, 0xed, 0xc4, 0x82, 0xd7, + 0xeb, 0x85, 0x12, 0x3e, 0xea, 0xd0, 0x07, 0xa0, 0x80, 0x48, 0xe9, 0xbd, + 0x9b, 0x3a, 0x8e, 0x8b, 0xa0, 0xfc, 0x07, 0xf0, 0x69, 0x4e, 0xc7, 0x1d, + 0xd9, 0x9a, 0x73, 0x18, 0x63, 0xb8, 0xe6, 0x4a, 0xa0, 0x81, 0xf0, 0xdb, + 0xb9, 0x88, 0xf4, 0x2b, 0x1f, 0x0d, 0xda, 0x31, 0xc0, 0xb0, 0x55, 0x79, + 0x56, 0x48, 0x22, 0xbb, 0x49, 0x7f, 0xb1, 0xf1, 0xf6, 0x6f, 0x42, 0xd3, + 0xba, 0x68, 0x3a, 0x8f, 0xe7, 0xac, 0x53, 0x30, 0x96, 0xec, 0x51, 0x7d, + 0xfc, 0xc0, 0x35, 0xe9, 0x59, 0xe7, 0x0e, 0xed, 0x29, 0x46, 0x50, 0x3c, + 0x4b, 0x36, 0xc6, 0x2a, 0xaa, 0x3b, 0xbe, 0xce, 0xd3, 0xda, 0x4d, 0x65, + 0xb0, 0xe8, 0x52, 0x68, 0xf0, 0x23, 0xde, 0x02, 0x77, 0xb3, 0xcc, 0xce, + 0x78, 0xdd, 0x8c, 0xf8, 0xbe, 0x5d, 0x0d, 0xa9, 0xb6, 0x96, 0x85, 0xbf, + 0x92, 0x2a, 0x6b, 0x1b, 0xe8, 0x76, 0x05, 0x13, 0x30, 0xd8, 0x3d, 0x80, + 0xaa, 0xa2, 0xa3, 0xbc, 0x07, 0xba, 0x9c, 0x75, 0x5b, 0x42, 0x03, 0xd8, + 0xde, 0x42, 0x44, 0xf7, 0x29, 0x43, 0x29, 0x0d, 0x48, 0x2b, 0x02, 0xd0, + 0xcc, 0xe9, 0x17, 0x47, 0x23, 0x73, 0x6d, 0xc5, 0x91, 0x6d, 0x4e, 0xc5, + 0xcf, 0xc3, 0x58, 0xaf, 0x6e, 0xa2, 0x9e, 0xe7, 0xe1, 0x88, 0xac, 0x62, + 0xff, 0xbc, 0x69, 0x57, 0xad, 0x0f, 0x08, 0xf8, 0x32, 0xfd, 0x79, 0xcb, + 0x30, 0xbc, 0xd2, 0xe5, 0x20, 0xd9, 0x0f, 0xd1, 0x33, 0xbf, 0xe4, 0x49, + 0x7a, 0x2b, 0x5c, 0xb3, 0x63, 0x13, 0x4d, 0xed, 0x17, 0xe7, 0x5b, 0xf4, + 0x36, 0x9d, 0x3c, 0x4e, 0x51, 0xb2, 0xf7, 0xf2, 0xcd, 0xfb, 0xec, 0x42, + 0x79, 0x46, 0xae, 0x18, 0x50, 0xdf, 0xbf, 0x5b, 0xb1, 0x9a, 0x49, 0x22, + 0xae, 0xe9, 0xf3, 0x86, 0x3f, 0xe0, 0xb4, 0xc6, 0x9c, 0x08, 0xd6, 0xd9, + 0xf4, 0x68, 0xbb, 0x33, 0x0e, 0x59, 0x3d, 0x76, 0xf0, 0xd7, 0x54, 0x04, + 0x19, 0x66, 0xee, 0x61, 0x11, 0x0d, 0x48, 0x10, 0x21, 0x16, 0x7c, 0xac, + 0x49, 0xab, 0xe0, 0x19, 0x85, 0x93, 0x48, 0x65, 0x7c, 0x5e, 0x6c, 0x1a, + 0xf5, 0xb0, 0xc6, 0x80, 0xa1, 0x2a, 0xd5, 0x71, 0x42, 0xec, 0x2f, 0x25, + 0xf7, 0xb8, 0x84, 0xcd, 0xf0, 0x5c, 0xcd, 0xee, 0x44, 0xcb, 0xeb, 0x74, + 0x96, 0x3c, 0xb0, 0x56, 0xcb, 0xaf, 0x7e, 0x9e, 0x4a, 0x12, 0x06, 0xae, + 0x57, 0x43, 0x2d, 0xb2, 0x11, 0x96, 0x05, 0xdb, 0xb3, 0x1a, 0x01, 0xa7, + 0x1d, 0x02, 0x81, 0x1c, 0x36, 0x41, 0x65, 0xf0, 0x67, 0xd6, 0xd0, 0x0f, + 0xec, 0x34, 0x7d, 0xd3, 0x89, 0xac, 0x60, 0x67, 0x95, 0x81, 0x84, 0xe7, + 0xbb, 0x9a, 0x59, 0x36, 0x3b, 0xde, 0xa4, 0x88, 0xda, 0xf2, 0xd2, 0xa2, + 0x0c, 0xba, 0xfb, 0x93, 0xbf, 0xc8, 0xad, 0xe8, 0x57, 0xa0, 0x2b, 0xbb, + 0x4e, 0xa9, 0x38, 0xe7, 0x86, 0x6b, 0x95, 0x34, 0x24, 0x96, 0xc0, 0x09, + 0xd9, 0xfd, 0x5f, 0x1c, 0x93, 0xd9, 0x72, 0xfa, 0xc4, 0x14, 0x72, 0x9c, + 0x19, 0x6f, 0xee, 0x12, 0x17, 0xee, 0x65, 0xb4, 0x8c, 0x83, 0x39, 0x3c, + 0x0f, 0xbf, 0x25, 0xcf, 0xee, 0x05, 0x8c, 0x6a, 0x56, 0x18, 0xf0, 0x20, + 0x72, 0xc1, 0xbf, 0xe4, 0xce, 0x37, 0xbf, 0x2b, 0xba, 0x70, 0x1e, 0xc2, + 0xc8, 0xcd, 0x58, 0xb9, 0x60, 0xc7, 0xfb, 0xd0, 0xce, 0xb9, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x7c, 0x63, 0x50, 0x90, + 0xcb, 0x9c, 0xce, 0x59, 0xb1, 0x47, 0xb0, 0x49, 0x9b, 0xfc, 0xfb, 0x3d, + 0x3d, 0x62, 0xcf, 0x58, 0x4c, 0x2a, 0x79, 0xf0, 0x72, 0x7f, 0x81, 0x41, + 0xac, 0x82, 0x2d, 0xa9, 0xf0, 0x0e, 0x4d, 0xd2, 0xe0, 0xbd, 0xca, 0x17, + 0xb7, 0x59, 0x9f, 0xdb, 0xfe, 0x51, 0x90, 0x88, 0xb9, 0xeb, 0x4e, 0xac, + 0x80, 0x30, 0x64, 0xc4, 0x49, 0xd1, 0xb6, 0x65, 0x67, 0xef, 0x9d, 0x5c, + 0x04, 0xe0, 0x9d, 0xbe, 0x47, 0x75, 0x9b, 0x6e, 0x30, 0x76, 0xad, 0x37, + 0x9a, 0x56, 0xff, 0xcd, 0x40, 0x26, 0x3e, 0xe2, 0x7d, 0x30, 0x55, 0x09, + 0x92, 0x25, 0x36, 0x2f, 0xf8, 0x55, 0xb8, 0x9b, 0x66, 0x49, 0x41, 0x9d, + 0x78, 0x6d, 0x3f, 0x54, 0x41, 0x01, 0x93, 0x9c, 0x5e, 0x0c, 0x4a, 0x38, + 0x79, 0x76, 0xb4, 0x98, 0xae, 0xf9, 0x99, 0x21, 0x05, 0x6a, 0xfb, 0xbc, + 0x44, 0xf7, 0xdc, 0x85, 0x5e, 0x5f, 0x18, 0x49, 0x22, 0x11, 0x6d, 0xa5, + 0x9e, 0x6b, 0x59, 0x60, 0xf8, 0x73, 0x8b, 0xcb, 0x38, 0xbb, 0xc9, 0xbf, + 0x49, 0x0e, 0x57, 0x65, 0x48, 0x41, 0x41, 0xa2, 0x40, 0x67, 0x91, 0x1d, + 0x54, 0xac, 0xa7, 0xef, 0x16, 0x8b, 0xc7, 0xd1, 0xe6, 0xdb, 0xc5, 0x9c, + 0xd4, 0x04, 0x67, 0xd8, 0x75, 0x21, 0x2b, 0x1d, 0x11, 0xc1, 0x79, 0x45, + 0xb4, 0x91, 0x7a, 0x97, 0x00, 0xde, 0xc6, 0xc5, 0x8a, 0xd1, 0xd7, 0xea, + 0xc1, 0x22, 0xe1, 0x58, 0x61, 0xf2, 0x89, 0x3d, 0xdb, 0x04, 0x3d, 0xe4, + 0xe9, 0xe7, 0xbf, 0x4b, 0x99, 0x8a, 0xc6, 0xf2, 0x09, 0xc4, 0xe2, 0x6d, + 0x0b, 0xda, 0x13, 0xfb, 0xff, 0xbf, 0x0b, 0xfc, 0x78, 0x33, 0xb8, 0x7b, + 0x3e, 0xd8, 0xba, 0x27, 0xba, 0xae, 0xdf, 0xce, 0xea, 0x80, 0x08, 0x38, + 0xd8, 0x33, 0x00, 0xa9, 0xb6, 0x88, 0x48, 0xa9, 0x3b, 0x54, 0xf0, 0x95, + 0xda, 0xba, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0xb1, 0xd7, 0x8d, 0x6c, 0xb9, 0x96, 0xdc, 0x64, 0x9b, 0x0c, 0x74, 0x54, + 0x59, 0x82, 0xf6, 0x6e, 0x7c, 0x4e, 0x23, 0x83, 0x04, 0x2e, 0x49, 0xfb, + 0x56, 0x4b, 0xcd, 0x0d, 0x76, 0x29, 0xb1, 0xce, 0x40, 0xa3, 0xd0, 0x02, + 0x16, 0x8e, 0x1c, 0x0a, 0x00, 0x5b, 0x8c, 0x06, 0xf9, 0x07, 0x97, 0x12, + 0x0c, 0x33, 0xd5, 0x48, 0x6d, 0xae, 0x7d, 0x2c, 0x8f, 0x74, 0x32, 0x24, + 0xcf, 0x91, 0xd7, 0xbe, 0xb2, 0x05, 0xcf, 0x2f, 0x93, 0xd5, 0x43, 0x90, + 0xce, 0x02, 0x97, 0xf8, 0x51, 0xb3, 0xba, 0x56, 0x5d, 0x94, 0x41, 0xa4, + 0x11, 0xf3, 0x21, 0xc0, 0xcc, 0x28, 0xf8, 0x5a, 0x00, 0x0a, 0xd4, 0x53, + 0xdd, 0xac, 0xfe, 0x25, 0x03, 0xea, 0x2b, 0x6b, 0x9d, 0x7e, 0x1a, 0xe1, + 0x5f, 0x5c, 0xa7, 0x47, 0xa2, 0x72, 0x4f, 0x92, 0x60, 0x25, 0x7c, 0x1c, + 0xa5, 0x34, 0xa6, 0x86, 0x0e, 0xda, 0x8f, 0x3f, 0xec, 0xe2, 0xe4, 0xad, + 0xa9, 0x41, 0xcc, 0x3d, 0x94, 0x43, 0xfd, 0x28, 0xd8, 0xb0, 0x0f, 0x05, + 0x9e, 0x2b, 0x27, 0x3f, 0xe0, 0x84, 0xbc, 0x9e, 0x7a, 0xa5, 0x83, 0x3d, + 0x3b, 0xac, 0x83, 0xd3, 0x16, 0x92, 0x8c, 0xd2, 0x4a, 0x81, 0xdd, 0xba, + 0x0a, 0xb7, 0xc5, 0x9f, 0x83, 0x0f, 0x78, 0xb8, 0xab, 0x2d, 0xca, 0xf8, + 0x6c, 0x06, 0xd7, 0x82, 0xb8, 0x61, 0x7d, 0x2a, 0x31, 0x3a, 0x39, 0x97, + 0x5f, 0xc7, 0x00, 0x6e, 0x46, 0xf2, 0xc5, 0x12, 0x71, 0x55, 0x5b, 0x10, + 0xaf, 0xbb, 0x07, 0x4c, 0x2f, 0xa3, 0x51, 0x53, 0x22, 0x20, 0xab, 0xed, + 0x02, 0x95, 0xc6, 0x5f, 0xaa, 0xb8, 0xc0, 0xcb, 0xe5, 0xe0, 0x25, 0x97, + 0xf7, 0xda, 0x1d, 0xd8, 0x5a, 0xff, 0x76, 0x0c, 0x3e, 0x33, 0x1b, 0x7a, + 0x15, 0xb8, 0x34, 0x75, 0xcf, 0xe9, 0xf3, 0x53, 0x61, 0x03, 0x2d, 0x52, + 0x29, 0x69, 0x3a, 0xc3, 0xd9, 0x22, 0xc0, 0x2d, 0x80, 0xed, 0x66, 0xc4, + 0xf4, 0x89, 0x60, 0x14, 0xdb, 0xec, 0x7d, 0xcc, 0x99, 0x5c, 0x94, 0x27, + 0xab, 0xed, 0xd2, 0x17, 0xf4, 0x36, 0xfc, 0x7e, 0x99, 0x98, 0xb6, 0x86, + 0xb6, 0x7c, 0x54, 0xd6, 0xec, 0xb5, 0xad, 0x62, 0xcc, 0xb0, 0xf7, 0x8c, + 0x52, 0x99, 0xf2, 0x44, 0x27, 0x3a, 0xb0, 0xff, 0x8f, 0x09, 0xae, 0xe1, + 0x61, 0xd8, 0x9f, 0xdd, 0x2f, 0x6b, 0xea, 0xd0, 0x12, 0x70, 0x8c, 0x9d, + 0x8f, 0x4c, 0x36, 0x98, 0x1e, 0x2e, 0xb5, 0x50, 0x63, 0x33, 0x9c, 0x4b, + 0xc3, 0xd4, 0xa0, 0xe6, 0x96, 0x96, 0x75, 0xfd, 0x8a, 0xc4, 0x0c, 0xa7, + 0xea, 0x9d, 0xf1, 0x23, 0x9e, 0x38, 0xff, 0x1a, 0x67, 0x36, 0x5f, 0x5f, + 0x17, 0x88, 0x1a, 0x43, 0x25, 0xea, 0x76, 0xb5, 0xcd, 0xce, 0x43, 0xf8, + 0x71, 0x2b, 0xdb, 0xf0, 0xcd, 0x76, 0xbd, 0x94, 0x57, 0xdb, 0x77, 0xcd, + 0xb2, 0x8f, 0xd1, 0xc0, 0xeb, 0x00, 0x61, 0x7f, 0x66, 0xb0, 0x43, 0x6e, + 0xe0, 0x9f, 0x11, 0x0e, 0x65, 0xf7, 0x4e, 0x00, 0x74, 0xc3, 0xeb, 0xb1, + 0xeb, 0x0c, 0x24, 0x5d, 0x15, 0x56, 0x16, 0x47, 0x87, 0xcf, 0x34, 0xbe, + 0x2a, 0xdd, 0x77, 0x55, 0xa4, 0x09, 0x15, 0x79, 0x8c, 0xaa, 0xce, 0x32, + 0x90, 0x9b, 0x16, 0x40, 0x94, 0x7f, 0x19, 0x27, 0xbc, 0xbf, 0x45, 0x4b, + 0xa5, 0xf0, 0xd0, 0x9e, 0x5b, 0xb9, 0x46, 0x6e, 0x72, 0x8f, 0x49, 0x3b, + 0x7a, 0xc1, 0x92, 0xb0, 0xd5, 0x25, 0x1b, 0x0b, 0xf3, 0xd0, 0x8a, 0x47, + 0x8b, 0xbe, 0xa4, 0xf9, 0x6a, 0x09, 0x84, 0x9a, 0x5b, 0x5b, 0xea, 0xbb, + 0x6f, 0xd8, 0xaf, 0xcd, 0x67, 0x9b, 0x79, 0x7c, 0x8f, 0xcc, 0xd7, 0x5f, + 0x3a, 0xc3, 0xd0, 0xb7, 0xba, 0x28, 0x83, 0x81, 0x4a, 0x05, 0x51, 0xaf, + 0xa0, 0x52, 0x34, 0xe3, 0x4f, 0xec, 0x82, 0xdc, 0x97, 0xd8, 0x69, 0xb2, + 0x0d, 0x68, 0x35, 0x87, 0x58, 0xc0, 0xcf, 0x58, 0x0d, 0xf6, 0x6b, 0x6d, + 0x2a, 0xc0, 0x72, 0xe4, 0x90, 0x8c, 0x7b, 0x45, 0xba, 0xf1, 0x13, 0x6f, + 0x8c, 0xd2, 0xdd, 0xc5, 0x8e, 0xc8, 0xec, 0xf9, 0xfb, 0xde, 0xe5, 0xaa, + 0xcb, 0xc0, 0xff, 0x77, 0x2d, 0x99, 0xb1, 0x69, 0x7f, 0xe3, 0x38, 0x61, + 0x35, 0xb6, 0x45, 0xdd, 0x73, 0x45, 0x84, 0x89, 0x1b, 0x96, 0x7e, 0x6a, + 0x1d, 0xd9, 0xe6, 0x76, 0xa8, 0x16, 0x0f, 0x42, 0xc9, 0x41, 0xec, 0x5d, + 0x25, 0x01, 0xb0, 0x45, 0xa6, 0xaa, 0x69, 0x87, 0x11, 0xa1, 0xb8, 0x9e, + 0x68, 0x48, 0x68, 0xe9, 0xb5, 0xc2, 0xff, 0x83, 0x8f, 0x71, 0xb9, 0xd7, + 0xbb, 0xae, 0x59, 0x8b, 0x1b, 0x4c, 0x44, 0xd8, 0xe3, 0xce, 0xab, 0x88, + 0xfb, 0x64, 0xd9, 0x61, 0x5a, 0x7d, 0xce, 0x3a, 0x27, 0xb5, 0xa3, 0xfd, + 0x5d, 0xa3, 0xb8, 0xa1, 0x15, 0x63, 0x0b, 0x75, 0x39, 0xc3, 0xa4, 0xfb, + 0x60, 0x53, 0xfd, 0x11, 0x21, 0x35, 0x0f, 0x19, 0x28, 0x14, 0xcd, 0x8a, + 0xcf, 0x33, 0xaa, 0x4f, 0x6a, 0x1e, 0x56, 0x87, 0xd5, 0x6e, 0x43, 0x9b, + 0xa3, 0x72, 0x95, 0x8c, 0x34, 0xa2, 0xac, 0x11, 0x76, 0x95, 0xd7, 0xdd, + 0xbf, 0x10, 0xf4, 0x0f, 0x2a, 0x64, 0xd2, 0x4d, 0x7b, 0xc6, 0x9b, 0x7d, + 0xf7, 0xa5, 0xb3, 0x84, 0x9a, 0x9a, 0x5e, 0xcf, 0x7f, 0x95, 0x6d, 0x44, + 0xd1, 0xb2, 0x19, 0xbb, 0xed, 0x37, 0x42, 0x4b, 0x4b, 0x6d, 0xb7, 0x10, + 0x02, 0x5f, 0x00, 0x1f, 0x24, 0xce, 0xb2, 0x8b, 0x3e, 0x7d, 0xc6, 0x6e, + 0x6c, 0x90, 0x75, 0xad, 0x3f, 0x9d, 0x63, 0x04, 0x76, 0x20, 0x7a, 0x56, + 0x48, 0xa1, 0x6a, 0x37, 0x74, 0xd2, 0xb7, 0x4f, 0xa3, 0x64, 0x62, 0xaa, + 0xce, 0x75, 0x8c, 0x15, 0x75, 0x79, 0xa0, 0xbd, 0xdd, 0x01, 0x46, 0xca, + 0xa0, 0x31, 0x1a, 0x16, 0x1f, 0xef, 0x8b, 0xc6, 0x54, 0x57, 0xfa, 0x6e, + 0x43, 0xdf, 0xb0, 0x99, 0xed, 0xa4, 0xcb, 0xeb, 0x91, 0x35, 0x14, 0x0c, + 0xa9, 0x1d, 0xb5, 0xa9, 0x32, 0x99, 0xe3, 0x89, 0x74, 0xaa, 0xa4, 0x65, + 0x1e, 0x82, 0x47, 0xfa, 0x37, 0x23, 0xe5, 0x86, 0xb6, 0xc0, 0xb6, 0x89, + 0x9a, 0xd9, 0xae, 0x29, 0x39, 0x7b, 0x66, 0xc7, 0x5b, 0x02, 0x08, 0x86, + 0xd4, 0xf0, 0x75, 0xc2, 0x05, 0x86, 0xc3, 0x75, 0xd2, 0x2a, 0x1e, 0xec, + 0x6e, 0x75, 0x29, 0x58, 0x8c, 0x25, 0x3b, 0x95, 0x21, 0xde, 0x42, 0xd5, + 0xb7, 0x15, 0x30, 0x09, 0x49, 0x78, 0x55, 0xd5, 0xf2, 0x30, 0x80, 0x93, + 0x8a, 0xce, 0x84, 0x27, 0xdb, 0x4a, 0x09, 0x30, 0x0c, 0x7f, 0x4d, 0xd1, + 0x0f, 0xda, 0x66, 0x58, 0xe1, 0x01, 0xfd, 0x75, 0x83, 0xf5, 0x39, 0x2e, + 0xe2, 0x6b, 0xde, 0xff, 0x20, 0x8a, 0xf7, 0xcc, 0x81, 0x8e, 0x99, 0xb4, + 0xeb, 0x76, 0x74, 0x38, 0x2b, 0xe0, 0x6d, 0x61, 0x8f, 0x39, 0x59, 0x10, + 0x7d, 0xb5, 0xd3, 0x14, 0x96, 0x04, 0x1d, 0x22, 0x89, 0xef, 0x15, 0x7c, + 0x28, 0x5a, 0xd6, 0x8d, 0xf3, 0xb7, 0x6a, 0x9a, 0xce, 0x21, 0x77, 0xfd, + 0x4f, 0x22, 0x26, 0x28, 0xb8, 0xb5, 0xb3, 0x73, 0xfd, 0x2a, 0x7b, 0x42, + 0x26, 0x77, 0x41, 0x93, 0xed, 0xf9, 0x8f, 0xa9, 0x92, 0xd5, 0x9f, 0x2e, + 0x60, 0xec, 0x60, 0x98, 0xf1, 0xd5, 0x11, 0xe2, 0xe0, 0xd7, 0x45, 0xa7, + 0xe4, 0xf2, 0x82, 0x61, 0x2f, 0x41, 0x1b, 0xd9, 0x8e, 0x78, 0xd5, 0x6b, + 0x68, 0x74, 0xf0, 0xc3, 0x83, 0x01, 0x16, 0x60, 0x6e, 0x34, 0x88, 0x45, + 0x8a, 0x86, 0x44, 0x5b, 0xa5, 0xa8, 0x55, 0xbc, 0xfa, 0x8f, 0xbd, 0x93, + 0x95, 0x3f, 0xab, 0x19, 0x54, 0x8f, 0x06, 0x8e, 0xca, 0x0b, 0x4a, 0x18, + 0x3f, 0x7a, 0x9c, 0x3f, 0xe6, 0xbe, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x81, 0x32, 0x41, 0x46, 0x59, 0x26, 0xf4, 0xef, + 0x93, 0x9f, 0x04, 0xc2, 0x67, 0x13, 0x32, 0x45, 0xc0, 0x79, 0x70, 0x27, + 0x21, 0x2b, 0xaf, 0x35, 0xf3, 0xc4, 0x88, 0x52, 0x28, 0xea, 0xca, 0x8a, + 0x08, 0x01, 0x6f, 0x61, 0xab, 0x10, 0xa3, 0xf0, 0x6b, 0x3b, 0x54, 0x64, + 0xf1, 0x63, 0x83, 0x38, 0x2b, 0x26, 0x18, 0x5a, 0x67, 0xc4, 0x67, 0x38, + 0x3f, 0x2c, 0x9a, 0xc9, 0x48, 0x33, 0x77, 0xb4, 0xb2, 0xc2, 0xc7, 0x08, + 0x21, 0x5e, 0xc4, 0x19, 0x59, 0xe1, 0xfa, 0x32, 0xa4, 0x4c, 0x3e, 0xba, + 0x65, 0x92, 0x98, 0x39, 0x71, 0x2f, 0x99, 0x08, 0xf8, 0xb3, 0x7a, 0x03, + 0x53, 0xd7, 0x68, 0xb2, 0x5e, 0xb0, 0xef, 0xe0, 0x1e, 0x7d, 0xb2, 0x23, + 0x5d, 0x2b, 0xd7, 0x09, 0xa6, 0x78, 0xa4, 0x7c, 0x08, 0xed, 0x8a, 0xf6, + 0x96, 0xa0, 0x10, 0x17, 0x62, 0x8b, 0x8a, 0xa0, 0xac, 0x22, 0x67, 0x02, + 0xa8, 0x66, 0x1a, 0xb5, 0x02, 0xde, 0xa5, 0xfa, 0x69, 0x29, 0x5f, 0x24, + 0x89, 0x46, 0x68, 0xd6, 0x51, 0x2a, 0xfe, 0x88, 0xf0, 0x40, 0xde, 0xd1, + 0x12, 0x2e, 0xed, 0x13, 0x7b, 0x49, 0xf6, 0xe1, 0x7a, 0xcf, 0x61, 0xcb, + 0x70, 0x9d, 0xaa, 0x51, 0x07, 0xc2, 0x54, 0x76, 0x89, 0x29, 0x94, 0x29, + 0x8b, 0x0e, 0xf5, 0xe8, 0x81, 0xc7, 0xdb, 0x59, 0x1e, 0x75, 0xda, 0x6a, + 0x94, 0x18, 0x16, 0xae, 0xbb, 0x43, 0x87, 0x56, 0x66, 0x8b, 0x84, 0xe9, + 0xa9, 0xd0, 0xd2, 0x8f, 0x5b, 0xbf, 0x1d, 0x24, 0x3a, 0xb7, 0x64, 0xff, + 0xe9, 0x22, 0x21, 0x65, 0xaf, 0x2b, 0x45, 0x8d, 0x28, 0xea, 0xbc, 0x07, + 0x10, 0x6e, 0xfb, 0x4d, 0x6f, 0x35, 0xe5, 0xeb, 0x5d, 0x29, 0x72, 0xe1, + 0x94, 0xad, 0xed, 0x25, 0xd7, 0x39, 0x63, 0x32, 0x37, 0x0b, 0xb2, 0xd7, + 0x54, 0x1f, 0xe4, 0x0d, 0xe7, 0xb3, 0xd1, 0xa6, 0x2a, 0xcf, 0x8e, 0x97, + 0xf1, 0xa8, 0xfc, 0xb1, 0x61, 0xdc, 0xb4, 0x8f, 0x29, 0xa2, 0x68, 0x4a, + 0xe6, 0x2f, 0x8a, 0x69, 0x2c, 0xa1, 0x1d, 0xe2, 0x9e, 0x65, 0x71, 0xb7, + 0x83, 0xef, 0x63, 0xf5, 0x36, 0xdc, 0xa0, 0x94, 0x5a, 0x45, 0x8a, 0x85, + 0x5e, 0x28, 0x86, 0x21, 0xd2, 0xbf, 0x7a, 0x2f, 0x76, 0x1c, 0x2a, 0x15, + 0xb2, 0xe8, 0xaf, 0x63, 0x37, 0xbe, 0xd8, 0x0a, 0xef, 0x54, 0xee, 0xe6, + 0xd9, 0xb3, 0xdb, 0x41, 0x55, 0xba, 0xd8, 0x14, 0x7c, 0x10, 0x61, 0x06, + 0x40, 0x45, 0x69, 0x37, 0x60, 0xf7, 0x6a, 0x7a, 0x23, 0x70, 0x30, 0x57, + 0x3e, 0xe5, 0x12, 0x24, 0xbc, 0x5e, 0x82, 0x89, 0xd8, 0x37, 0xc9, 0x33, + 0xb9, 0x38, 0xa5, 0xba, 0xed, 0xdd, 0x93, 0x58, 0x81, 0x15, 0xec, 0x15, + 0x70, 0x2f, 0x30, 0xfa, 0xaf, 0xf7, 0xf5, 0xcb, 0x41, 0x74, 0xea, 0xc0, + 0x91, 0xbe, 0x53, 0x4c, 0xc2, 0x74, 0x1b, 0x5b, 0x8c, 0x74, 0xd8, 0xc3, + 0x4a, 0x12, 0xaa, 0x57, 0xd6, 0x61, 0xb1, 0xb8, 0x81, 0x5d, 0x81, 0x37, + 0x1e, 0x5b, 0x3d, 0x5a, 0xbc, 0xa6, 0xb2, 0x27, 0xe3, 0x01, 0x4c, 0xf0, + 0xad, 0x7b, 0xdf, 0x50, 0xf9, 0xd7, 0xb7, 0xcc, 0xa8, 0x5c, 0x3d, 0x9a, + 0xb7, 0x60, 0x3e, 0x63, 0x3f, 0x6a, 0x08, 0x0b, 0x82, 0xdc, 0x3e, 0xfa, + 0x24, 0x33, 0xd3, 0x01, 0xbf, 0xef, 0xeb, 0x52, 0x3f, 0x91, 0x61, 0xda, + 0xe2, 0x26, 0x10, 0xdf, 0xe4, 0x9b, 0x77, 0x91, 0x22, 0xc5, 0x4e, 0x9c, + 0x0b, 0x32, 0xff, 0x27, 0x85, 0x85, 0x0c, 0x99, 0x50, 0x8f, 0xad, 0x5d, + 0x06, 0x18, 0x52, 0xb4, 0x64, 0x09, 0xc4, 0xa4, 0x84, 0xd4, 0x81, 0x07, + 0x0a, 0x97, 0x55, 0xf8, 0x96, 0x52, 0xb2, 0x9a, 0xf4, 0x06, 0x2c, 0x9a, + 0x3b, 0x8b, 0xaa, 0x67, 0x18, 0x3a, 0xee, 0xbc, 0xca, 0x8f, 0x46, 0xf6, + 0x4a, 0x33, 0x5b, 0x56, 0x09, 0xb2, 0x72, 0x87, 0xdb, 0xbb, 0x57, 0x67, + 0x53, 0x82, 0x77, 0x31, 0x66, 0xbb, 0xf1, 0x33, 0x6d, 0x55, 0x82, 0xaa, + 0x80, 0xd4, 0x4d, 0xb8, 0xab, 0xbd, 0x2a, 0xda, 0x10, 0x3a, 0xc8, 0xf0, + 0x14, 0x1e, 0xcb, 0x8e, 0x76, 0x6c, 0xc8, 0x74, 0x05, 0xb3, 0x51, 0xbd, + 0x63, 0x06, 0x69, 0x05, 0x2a, 0x21, 0xd6, 0x2f, 0xe4, 0x38, 0xae, 0xf8, + 0xd4, 0xe9, 0xa7, 0xe8, 0xc8, 0x5a, 0x65, 0x7d, 0x54, 0x34, 0x33, 0x0d, + 0xf6, 0x07, 0xd6, 0x8c, 0xe5, 0x72, 0x9b, 0xfb, 0x60, 0x49, 0xd2, 0xaf, + 0xb4, 0x17, 0xc4, 0x74, 0x8d, 0xe5, 0x54, 0xda, 0x96, 0x56, 0x7d, 0x97, + 0x62, 0xe8, 0xec, 0x0d, 0x2b, 0x02, 0x2e, 0x59, 0xf8, 0xa1, 0x06, 0x6a, + 0xb6, 0x3e, 0x15, 0xeb, 0x64, 0x1a, 0x48, 0x3d, 0x53, 0x2c, 0x42, 0x3b, + 0x97, 0xa1, 0x3f, 0x47, 0x8b, 0x74, 0x87, 0x8b, 0x96, 0x63, 0x08, 0x4c, + 0x99, 0x38, 0x5a, 0xb6, 0x93, 0xa8, 0xcc, 0xee, 0x62, 0x3a, 0x00, 0x6d, + 0x5c, 0xab, 0x77, 0x3c, 0x46, 0xae, 0x6e, 0xeb, 0xf1, 0xf9, 0x63, 0xf1, + 0xa2, 0x31, 0x21, 0x38, 0xc3, 0x4f, 0xe2, 0x3a, 0x33, 0x7f, 0xe7, 0xc6, + 0x69, 0xd5, 0x1c, 0x7e, 0x5b, 0x4f, 0xb1, 0x50, 0x3b, 0xbe, 0x31, 0xa7, + 0x42, 0xa3, 0x97, 0x7b, 0xe3, 0x90, 0xd0, 0x07, 0xfd, 0x05, 0xb9, 0xf2, + 0x47, 0xc4, 0xc8, 0xdd, 0x1c, 0x3c, 0xa4, 0x22, 0x96, 0x04, 0xca, 0x28, + 0x17, 0xcc, 0x5c, 0x49, 0x7e, 0xc6, 0x93, 0x98, 0xd3, 0x8b, 0xd2, 0xf6, + 0x4a, 0xb6, 0xbe, 0x8d, 0xa2, 0xdd, 0xb6, 0x7c, 0x66, 0x0c, 0x29, 0xcb, + 0x1d, 0x98, 0xf6, 0xe4, 0xe5, 0x30, 0x4c, 0x84, 0xbf, 0x6f, 0x71, 0x4e, + 0xc2, 0x12, 0x9f, 0x35, 0xd6, 0xf8, 0xc6, 0x30, 0xe9, 0x9e, 0x1a, 0x8a, + 0x2f, 0xd1, 0x96, 0xb3, 0x3c, 0x0f, 0xf5, 0x78, 0xa7, 0xe0, 0xbd, 0x4b, + 0xe0, 0xd8, 0x3d, 0x57, 0xa5, 0x44, 0xa0, 0xd9, 0x10, 0x79, 0xd2, 0x10, + 0x50, 0xc7, 0x77, 0x73, 0x09, 0xf8, 0xb4, 0xcf, 0x66, 0xe3, 0x0c, 0xfb, + 0x96, 0xf8, 0x52, 0xb3, 0x7e, 0x44, 0xf0, 0x03, 0x54, 0xd4, 0xa2, 0x57, + 0x38, 0x8a, 0x96, 0xfc, 0x7c, 0x4c, 0x9f, 0x3a, 0xf2, 0xa2, 0x48, 0xbb, + 0x3e, 0xd1, 0x11, 0x2c, 0xab, 0xdf, 0x53, 0x96, 0xac, 0x58, 0x33, 0xb9, + 0xdd, 0xd2, 0x4f, 0x8a, 0x0a, 0x89, 0x0e, 0xd3, 0x6f, 0x58, 0x8c, 0xa1, + 0x0a, 0x0b, 0xa7, 0xd7, 0x1f, 0x0a, 0x70, 0xe3, 0x43, 0x12, 0x56, 0xb8, + 0x6c, 0xf8, 0x75, 0x4e, 0x2b, 0xb0, 0x17, 0x29, 0xe4, 0x95, 0x85, 0xd8, + 0x85, 0x95, 0x63, 0x55, 0xa8, 0x82, 0xf0, 0xe7, 0x7d, 0xf3, 0xf1, 0x78, + 0x66, 0xd1, 0x92, 0x71, 0x99, 0xad, 0x30, 0x94, 0xe9, 0x54, 0x2c, 0xe1, + 0x57, 0xf3, 0x6a, 0xe6, 0x0c, 0x5e, 0xc7, 0x58, 0xba, 0xb7, 0x61, 0xd3, + 0x74, 0x72, 0x96, 0x06, 0x0b, 0x01, 0x3d, 0xc2, 0xa1, 0xb4, 0x38, 0x81, + 0x19, 0x44, 0xbc, 0x84, 0x52, 0x22, 0xc9, 0x67, 0x81, 0x99, 0xfb, 0x0a, + 0xc2, 0xff, 0x50, 0x67, 0xbe, 0x38, 0x5e, 0x13, 0x16, 0x60, 0x83, 0x35, + 0xb9, 0x2f, 0xa9, 0x55, 0xbb, 0x30, 0x6b, 0x19, 0xfc, 0x2a, 0x40, 0x24, + 0x74, 0x20, 0x57, 0x78, 0xb9, 0x55, 0xb7, 0x70, 0x86, 0x65, 0x43, 0x1c, + 0x76, 0x2e, 0x91, 0x83, 0x5e, 0x33, 0xc2, 0xd4, 0xcc, 0xb5, 0x1c, 0x45, + 0xaf, 0xa3, 0x87, 0x95, 0x9b, 0x77, 0x50, 0x44, 0x7e, 0xdd, 0xca, 0x3f, + 0x51, 0x21, 0xae, 0xf2, 0x15, 0xa9, 0x32, 0x94, 0xca, 0xde, 0x3b, 0x97, + 0x13, 0x6b, 0xff, 0xe0, 0x79, 0x39, 0x40, 0xf0, 0x66, 0x7d, 0x5e, 0xef, + 0xec, 0x0a, 0x35, 0xd2, 0x0d, 0x09, 0x19, 0x13, 0xf2, 0xc2, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xdc, 0x07, 0x2e, 0x46, + 0xab, 0x4d, 0x6d, 0xf7, 0x24, 0xba, 0x02, 0xe3, 0xc5, 0xe3, 0xed, 0x64, + 0xc6, 0x77, 0x5a, 0x14, 0xae, 0x38, 0x52, 0x8c, 0x16, 0x2c, 0x52, 0x0e, + 0xf6, 0x65, 0x99, 0xcc, 0xf6, 0x9f, 0x77, 0xcc, 0x2e, 0xaf, 0x14, 0xd1, + 0xf0, 0x0f, 0xa7, 0x3e, 0x5b, 0x74, 0xff, 0xb9, 0xd3, 0x30, 0x02, 0x5e, + 0x52, 0xc8, 0x6f, 0x57, 0xef, 0x28, 0xf5, 0xfa, 0x9e, 0x70, 0x00, 0xfc, + 0x3e, 0xc3, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0xaa, 0x9f, 0x86, 0xb0, 0x6d, 0xa1, 0x0c, 0xfa, 0xef, 0xb3, 0x6a, 0x50, + 0xa6, 0xfe, 0xff, 0xa9, 0x61, 0x0b, 0x18, 0x72, 0xee, 0xc6, 0xcd, 0x3a, + 0x34, 0x5e, 0xa8, 0x81, 0x31, 0x54, 0x25, 0x05, 0xc1, 0xd9, 0x66, 0x3d, + 0x17, 0xbb, 0x03, 0x21, 0x07, 0x69, 0x3a, 0x37, 0xe8, 0xd4, 0x6a, 0x68, + 0xe1, 0xa3, 0x19, 0x5a, 0x8d, 0x14, 0x11, 0x09, 0xef, 0xae, 0xfe, 0x94, + 0x19, 0x8a, 0xe4, 0xb9, 0x6e, 0xe8, 0xfa, 0x12, 0x2a, 0x5d, 0x00, 0x29, + 0x27, 0x6d, 0x5a, 0xa5, 0x09, 0x34, 0x79, 0x2b, 0xa8, 0xcc, 0x42, 0xb4, + 0xde, 0xe0, 0x91, 0xb9, 0x06, 0x0c, 0x11, 0x17, 0x25, 0x7a, 0x35, 0x57, + 0x51, 0x40, 0xf3, 0xc7, 0xc6, 0x4a, 0x69, 0x98, 0x2b, 0x2b, 0x3e, 0x5d, + 0x32, 0xd8, 0x8f, 0xb0, 0x1d, 0xee, 0x77, 0xe3, 0xaf, 0x4f, 0x71, 0x05, + 0x04, 0xd2, 0xff, 0x51, 0xed, 0xa4, 0x69, 0x50, 0x24, 0x2a, 0xe5, 0xaa, + 0xbb, 0xc6, 0x7a, 0x7f, 0xb2, 0xdf, 0x1d, 0xc2, 0x02, 0x2e, 0x52, 0xd1, + 0xd9, 0x5b, 0xe7, 0x6c, 0x50, 0x31, 0x4e, 0xdf, 0x8e, 0x3f, 0x37, 0xfc, + 0xf5, 0x34, 0x0e, 0xdb, 0x4c, 0x5d, 0x7d, 0xc8, 0xe4, 0x72, 0x40, 0xcb, + 0x95, 0xa5, 0x41, 0xeb, 0x78, 0x5f, 0x64, 0x20, 0x55, 0x19, 0xc7, 0xf9, + 0x9c, 0x71, 0x40, 0x8f, 0xcc, 0x2d, 0x86, 0xc0, 0xf4, 0x36, 0x2b, 0x0e, + 0x28, 0xb4, 0xad, 0x1b, 0xde, 0x60, 0x67, 0x03, 0x0f, 0x7c, 0x18, 0xd9, + 0xc3, 0x73, 0x67, 0x0d, 0x44, 0x3d, 0xbe, 0x7c, 0xcf, 0x96, 0x22, 0x0b, + 0x0e, 0x3a, 0x0b, 0xcf, 0x04, 0x95, 0x92, 0x7d, 0x4b, 0xa2, 0x6a, 0x0b, + 0x47, 0x72, 0x73, 0xa8, 0x9b, 0x96, 0x3d, 0xc6, 0x03, 0x34, 0xb1, 0x69, + 0xc2, 0x50, 0x60, 0x89, 0x8c, 0x55, 0x8f, 0x8e, 0x74, 0xa8, 0x9e, 0x25, + 0xe4, 0x0e, 0x73, 0xef, 0x4f, 0x51, 0xbe, 0xed, 0x5c, 0x14, 0xd3, 0xfa, + 0x94, 0x58, 0x8d, 0x5c, 0xa0, 0xb1, 0xfc, 0x37, 0x6e, 0x9c, 0x9e, 0x61, + 0xe5, 0x12, 0x13, 0xb2, 0x88, 0xc6, 0xcf, 0x60, 0x3f, 0x0d, 0x51, 0x33, + 0x22, 0xfa, 0xfb, 0x2d, 0x2b, 0x8d, 0x43, 0x9b, 0x3d, 0x1e, 0x88, 0x24, + 0x50, 0x78, 0xf7, 0x7e, 0x45, 0xb1, 0x0f, 0xa9, 0xe6, 0x77, 0xf8, 0x78, + 0xff, 0x57, 0x6a, 0x05, 0x06, 0x0c, 0x7e, 0x1e, 0x7f, 0xe9, 0x90, 0xe8, + 0x61, 0x68, 0xbc, 0x9e, 0xc4, 0xe5, 0x06, 0x04, 0x76, 0xcc, 0x01, 0x57, + 0x1a, 0x55, 0x9e, 0x45, 0x26, 0xd6, 0xd8, 0xc2, 0x50, 0x25, 0xfc, 0x72, + 0x4e, 0x18, 0xbe, 0xf2, 0x2f, 0xc0, 0x1b, 0xc8, 0x14, 0xeb, 0x24, 0xda, + 0x15, 0x0a, 0x83, 0x38, 0xc5, 0xdd, 0xc9, 0xd7, 0x12, 0x35, 0x55, 0xdf, + 0x2c, 0x23, 0xea, 0x17, 0xca, 0xbf, 0x18, 0xc9, 0x80, 0x63, 0x4b, 0x77, + 0x8b, 0x17, 0x01, 0x05, 0x1b, 0xa3, 0x0b, 0x0f, 0xdd, 0xc6, 0xe0, 0xdf, + 0xc9, 0xa6, 0x8c, 0x50, 0x95, 0x8d, 0x6c, 0x96, 0x67, 0xff, 0x88, 0x38, + 0x3b, 0x76, 0x72, 0x11, 0x35, 0xa0, 0x1c, 0xc8, 0x96, 0x9c, 0xe5, 0x90, + 0x79, 0x0e, 0x62, 0x57, 0x00, 0xd9, 0x57, 0xf8, 0xa4, 0xc2, 0xc2, 0x0a, + 0x17, 0x8e, 0xd7, 0x03, 0x6d, 0x4d, 0x14, 0xb6, 0x96, 0x8a, 0x76, 0x67, + 0x58, 0xce, 0x9c, 0xb3, 0x10, 0x49, 0x06, 0xeb, 0x56, 0x43, 0x40, 0xcb, + 0xd4, 0xd7, 0x59, 0x42, 0xa4, 0xd7, 0x21, 0x6a, 0x51, 0x3d, 0x1c, 0x54, + 0xd7, 0xd6, 0xa2, 0xcf, 0xf8, 0xf6, 0x72, 0x35, 0x04, 0xa6, 0xe3, 0x53, + 0xca, 0xc5, 0x62, 0xee, 0xa9, 0xc3, 0x6d, 0x1b, 0xc4, 0xc5, 0xd9, 0xa7, + 0x37, 0xc2, 0x04, 0x01, 0xc9, 0x4a, 0x2e, 0x26, 0xdd, 0x12, 0x6e, 0x41, + 0x64, 0xb4, 0xe8, 0xe8, 0xc7, 0xf8, 0xab, 0x8a, 0xab, 0x1d, 0x7f, 0x2d, + 0x58, 0xc2, 0xc4, 0xf0, 0x5d, 0x11, 0x35, 0x52, 0x88, 0xbc, 0x0f, 0x44, + 0x6e, 0x91, 0x1e, 0x87, 0xb4, 0xb1, 0x91, 0x52, 0x32, 0xe4, 0x38, 0x6d, + 0x5e, 0x8d, 0x30, 0xf0, 0xbc, 0xc3, 0x15, 0x80, 0x47, 0x36, 0x35, 0xb0, + 0x93, 0xf3, 0xc4, 0x82, 0xc7, 0x73, 0xc1, 0x67, 0x0c, 0x7a, 0x31, 0x36, + 0xbc, 0x73, 0x67, 0x66, 0xae, 0x48, 0x82, 0x27, 0x6e, 0x14, 0xd0, 0xd5, + 0x12, 0x10, 0xce, 0x5e, 0x37, 0xcd, 0x7e, 0xa5, 0xcb, 0xff, 0x91, 0xf0, + 0x62, 0xdb, 0x95, 0x74, 0x0c, 0x8c, 0x1e, 0x78, 0x11, 0x02, 0xb3, 0x02, + 0x0b, 0x31, 0xe7, 0x4e, 0x8b, 0x58, 0x6a, 0xde, 0x20, 0x93, 0x8b, 0x8e, + 0x62, 0x03, 0x24, 0xc9, 0xca, 0xf8, 0x44, 0x1d, 0x0c, 0x1b, 0xd8, 0x5d, + 0xcc, 0xe2, 0x8e, 0x02, 0xc6, 0x5c, 0x06, 0x45, 0xe6, 0x94, 0x8f, 0xa2, + 0x3e, 0xf5, 0xe9, 0xf5, 0x88, 0x87, 0xb2, 0x84, 0x1e, 0xb6, 0xb6, 0xfc, + 0x9f, 0x8e, 0x79, 0xf5, 0x4b, 0x24, 0x81, 0x3e, 0x5d, 0xf4, 0x10, 0x6e, + 0xdd, 0x8c, 0x8c, 0xae, 0xc6, 0x2c, 0x26, 0xb2, 0xfc, 0xf3, 0x99, 0xe8, + 0x8c, 0x65, 0x5d, 0x6c, 0xa8, 0x1d, 0x6f, 0x1e, 0x32, 0x0a, 0xee, 0x87, + 0xf6, 0xe1, 0xdd, 0x5e, 0x7f, 0x7a, 0x90, 0x8c, 0x3f, 0xe8, 0x47, 0x95, + 0x9b, 0xc8, 0x2c, 0x49, 0xc9, 0xe4, 0x2d, 0xea, 0x58, 0xfc, 0x29, 0x1a, + 0xb7, 0xa1, 0xf9, 0xb8, 0x84, 0x41, 0xa0, 0xf1, 0x77, 0x83, 0x56, 0x73, + 0x86, 0xea, 0xf4, 0xf5, 0x2a, 0xa6, 0x6b, 0x00, 0x64, 0x39, 0x08, 0x8f, + 0xf0, 0x22, 0x1a, 0x4c, 0xf2, 0x5a, 0xd0, 0xaa, 0x39, 0xae, 0x8a, 0xbc, + 0x03, 0x99, 0xf7, 0xcc, 0x80, 0xdf, 0x2b, 0x85, 0xbe, 0x1a, 0x97, 0x28, + 0x63, 0x04, 0x72, 0x75, 0x75, 0xb4, 0x9c, 0xd3, 0x17, 0xcc, 0x1e, 0xa1, + 0xd2, 0x47, 0x18, 0x45, 0xad, 0xb4, 0x0a, 0x32, 0x31, 0x36, 0x64, 0x48, + 0x3f, 0x7b, 0x4b, 0xc0, 0xd6, 0x78, 0x46, 0xaa, 0x90, 0x89, 0xf9, 0x36, + 0x3d, 0xb4, 0xb3, 0x50, 0x51, 0xd9, 0x55, 0x6f, 0xa9, 0xe7, 0x25, 0xaf, + 0xa0, 0xca, 0x9d, 0x45, 0x83, 0xc3, 0x0b, 0x2a, 0x0c, 0xf9, 0x3f, 0xe4, + 0x08, 0xf4, 0xbd, 0x23, 0x45, 0x85, 0xcf, 0x41, 0x93, 0xd3, 0x21, 0x5f, + 0x53, 0xa2, 0x5b, 0xa9, 0xf5, 0xe9, 0x8f, 0x2a, 0x2d, 0x53, 0x3c, 0x36, + 0x17, 0xce, 0x37, 0x35, 0x3e, 0x9e, 0x6b, 0xbc, 0xba, 0xaa, 0xa5, 0x61, + 0x79, 0x98, 0x8e, 0xbd, 0x19, 0xf4, 0x5f, 0xa9, 0xb8, 0x96, 0xa2, 0xce, + 0x32, 0x00, 0xab, 0x51, 0xcb, 0xfa, 0x30, 0x3a, 0x83, 0x92, 0x91, 0xad, + 0x08, 0x61, 0x62, 0x51, 0x7f, 0x19, 0xa9, 0x2a, 0x84, 0xf2, 0xab, 0x7e, + 0x5e, 0xa7, 0x5a, 0x54, 0x7f, 0x68, 0x2a, 0x7b, 0x4f, 0xde, 0x45, 0x1d, + 0xef, 0x73, 0x5f, 0xc0, 0x40, 0x6e, 0xec, 0x6c, 0xe9, 0xa5, 0x6b, 0x46, + 0x54, 0x7c, 0x24, 0x8b, 0xa4, 0xe5, 0xb4, 0x82, 0x31, 0x1f, 0x3e, 0x79, + 0x2e, 0x21, 0x8c, 0xf1, 0xbd, 0xad, 0x7c, 0x28, 0xcc, 0xbd, 0x58, 0x72, + 0xe9, 0x6a, 0x04, 0x56, 0x67, 0x0f, 0x62, 0x98, 0x5a, 0x97, 0x4b, 0xe2, + 0x67, 0x70, 0xbb, 0x17, 0xb1, 0x84, 0x5b, 0xd4, 0x6e, 0xab, 0x90, 0x29, + 0x20, 0x93, 0x34, 0xa8, 0x03, 0x0f, 0xed, 0x1a, 0xf0, 0x1b, 0x92, 0x87, + 0x43, 0xa5, 0x6a, 0x1c, 0xdc, 0xd7, 0x22, 0x68, 0x83, 0x98, 0x74, 0x2a, + 0x4c, 0x51, 0xef, 0x71, 0x19, 0xd5, 0x3d, 0x05, 0x19, 0x61, 0xb2, 0x52, + 0xa8, 0x6e, 0xda, 0x72, 0x51, 0x66, 0x9f, 0xf0, 0x12, 0xf6, 0x18, 0x60, + 0xcc, 0xd7, 0x2f, 0x2e, 0x83, 0x14, 0x09, 0xdb, 0x55, 0x1c, 0xf2, 0xaf, + 0xfd, 0xa4, 0x40, 0xf1, 0x4a, 0xc7, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x9c, 0x52, 0xff, 0x48, 0x06, 0x61, 0x76, 0x6d, + 0xd7, 0x44, 0xb1, 0x0c, 0x32, 0x62, 0x15, 0xa1, 0xc3, 0x97, 0x03, 0xdd, + 0xed, 0x20, 0x3c, 0x3a, 0x09, 0x16, 0xe5, 0x7d, 0x8c, 0xf9, 0x7b, 0x22, + 0x5e, 0x3a, 0xdd, 0xf0, 0xc6, 0xf0, 0x3a, 0xd4, 0x94, 0x85, 0x1c, 0x60, + 0x74, 0x91, 0xa3, 0xe2, 0x8a, 0xe5, 0x3e, 0xd4, 0x95, 0x28, 0x8b, 0x1a, + 0x7b, 0xbe, 0x07, 0xc0, 0xe3, 0x6b, 0xb9, 0x85, 0x82, 0x0b, 0x24, 0xba, + 0x1c, 0xfc, 0xc0, 0x0a, 0x21, 0x33, 0xad, 0x00, 0x19, 0xce, 0xb5, 0x8f, + 0x73, 0x05, 0xf1, 0xac, 0x03, 0xbe, 0x1f, 0x22, 0xd5, 0x32, 0x5e, 0x50, + 0xe3, 0xe0, 0x62, 0x26, 0xf4, 0xb0, 0x85, 0xd8, 0xf7, 0xa7, 0xf4, 0xa7, + 0xff, 0x10, 0xb8, 0xbc, 0xe0, 0x3e, 0x4d, 0xcb, 0x37, 0x74, 0xcc, 0x85, + 0xed, 0xa0, 0x34, 0x6c, 0xfa, 0x37, 0x84, 0x6a, 0x94, 0x55, 0x3b, 0x1e, + 0x14, 0xab, 0x26, 0x7b, 0x3e, 0xac, 0xc3, 0x79, 0xcd, 0x1b, 0x00, 0x02, + 0xb3, 0x01, 0xc3, 0x10, 0xdd, 0x56, 0x7d, 0x0e, 0x69, 0x39, 0x3c, 0x17, + 0xa3, 0xae, 0x9c, 0x2d, 0xc7, 0x5a, 0x0b, 0x7c, 0xd0, 0xac, 0xa1, 0x91, + 0x6a, 0x6d, 0xc0, 0x3f, 0x98, 0xf1, 0x21, 0xf5, 0xa5, 0x7c, 0xbc, 0x70, + 0x0d, 0x7b, 0x2f, 0x0d, 0x5a, 0xa5, 0x4a, 0x5a, 0xff, 0x51, 0xbf, 0x7f, + 0xb5, 0x4f, 0x2c, 0xba, 0xa9, 0x46, 0x81, 0x6b, 0xac, 0xc6, 0x62, 0x2d, + 0xd7, 0xb5, 0x04, 0x5f, 0xd4, 0x5f, 0x1f, 0x6b, 0x11, 0x7d, 0xe3, 0x58, + 0x1f, 0xb5, 0xbf, 0x16, 0x43, 0x88, 0x05, 0xf5, 0xa4, 0x7b, 0xb5, 0x0e, + 0xf4, 0x01, 0xb6, 0x90, 0x69, 0x52, 0x0a, 0x5e, 0x9b, 0x87, 0x51, 0x5e, + 0xd5, 0xed, 0x2c, 0xcc, 0x58, 0xad, 0xe6, 0x77, 0xa2, 0xc5, 0x7c, 0x1e, + 0xc5, 0x92, 0xbe, 0xed, 0x3a, 0x9a, 0x97, 0xed, 0x56, 0xc8, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x16, 0xe8, 0x24, 0xe3, + 0x82, 0x36, 0x8e, 0x50, 0x45, 0xbe, 0xc6, 0x10, 0x02, 0xb9, 0x6d, 0xf9, + 0xed, 0x8f, 0x64, 0x35, 0x4d, 0x2c, 0x9f, 0x99, 0xdc, 0xee, 0xfa, 0x63, + 0x99, 0xc4, 0xb8, 0x3d, 0x77, 0xea, 0xda, 0xd5, 0x95, 0x8b, 0x8e, 0x76, + 0x02, 0x9c, 0x62, 0xa0, 0xad, 0xfe, 0x80, 0x61, 0x72, 0x59, 0xd6, 0x9f, + 0x16, 0x2e, 0x09, 0x71, 0xb8, 0xd7, 0x65, 0x25, 0xc2, 0x5b, 0x40, 0x67, + 0x8e, 0xd6, 0xf8, 0xdf, 0x67, 0x29, 0x19, 0xa2, 0xa6, 0x07, 0xf3, 0xc8, + 0x91, 0x7d, 0xf2, 0x50, 0x71, 0xba, 0x5c, 0x2d, 0xa7, 0xae, 0xc4, 0xd5, + 0xeb, 0xb9, 0x0d, 0x2d, 0x23, 0xe5, 0x8c, 0x65, 0xf5, 0xf8, 0x97, 0x69, + 0xde, 0x25, 0x6f, 0xea, 0x12, 0x72, 0x3e, 0xb9, 0xa7, 0x8d, 0xcf, 0xa5, + 0x66, 0xee, 0x4e, 0x2e, 0x66, 0x6b, 0xec, 0x77, 0x7f, 0x53, 0xdc, 0x29, + 0x73, 0x5e, 0xe9, 0x2f, 0x79, 0xac, 0x8d, 0x0f, 0x44, 0x09, 0x5d, 0x25, + 0x1d, 0x78, 0xb6, 0xe9, 0xd0, 0xfa, 0x8f, 0x5f, 0x9c, 0xf0, 0xe0, 0xfc, + 0x62, 0x9f, 0x52, 0x6b, 0x5b, 0x8e, 0x3f, 0xdf, 0xb4, 0xf1, 0xdf, 0x35, + 0xd0, 0x8f, 0x5a, 0xc9, 0x1f, 0x08, 0x86, 0xaa, 0x5a, 0x9e, 0xe8, 0xb0, + 0xaa, 0xd4, 0xcd, 0x2a, 0x5b, 0x4f, 0x7f, 0x39, 0x9f, 0x7f, 0x21, 0xf2, + 0xfd, 0x05, 0x96, 0x53, 0x09, 0xfd, 0x36, 0x4c, 0xcd, 0x98, 0x74, 0xf5, + 0xbd, 0xcd, 0x9e, 0x14, 0x15, 0x05, 0xb9, 0x3d, 0x5f, 0x8a, 0x02, 0x86, + 0x10, 0xd7, 0xd4, 0x01, 0x20, 0xd9, 0x8c, 0x65, 0x7d, 0x9d, 0x39, 0x25, + 0xbc, 0xce, 0x1a, 0xb1, 0x76, 0x92, 0xc3, 0x03, 0xed, 0xa2, 0x41, 0x31, + 0x0d, 0xc0, 0x40, 0x94, 0x01, 0xbc, 0x9b, 0xe9, 0x5e, 0x3e, 0x8c, 0x49, + 0xf6, 0x98, 0x0c, 0x39, 0x79, 0xdc, 0xd1, 0x1b, 0xc5, 0xb2, 0x20, 0xb4, + 0x6c, 0xb4, 0x4f, 0xce, 0xf4, 0x6c, 0x0b, 0xef, 0x85, 0xf2, 0x7d, 0x9a, + 0x90, 0x58, 0x1b, 0x51, 0x56, 0x52, 0xac, 0x75, 0x9f, 0x17, 0xe6, 0x48, + 0xaf, 0x18, 0x4c, 0xd8, 0x67, 0xe8, 0xd2, 0x61, 0xbc, 0xa0, 0x95, 0xc9, + 0x78, 0xd8, 0xa2, 0x1d, 0x47, 0x59, 0x30, 0xcf, 0xf3, 0x79, 0x06, 0xd4, + 0x25, 0xf8, 0x9c, 0x5c, 0x28, 0xee, 0xb0, 0xd2, 0xb6, 0xaf, 0x34, 0x0e, + 0xe5, 0xe4, 0x16, 0x2e, 0x05, 0x45, 0x23, 0xc1, 0x88, 0x90, 0x4a, 0x8f, + 0xff, 0xfb, 0xe2, 0xc0, 0xb7, 0xae, 0xb5, 0x50, 0xc9, 0x26, 0xf0, 0xa2, + 0xf5, 0x21, 0x23, 0x79, 0x23, 0xb6, 0x8f, 0x57, 0x64, 0xd1, 0x27, 0xc2, + 0x07, 0x63, 0xa6, 0x54, 0x1f, 0x2f, 0xca, 0x16, 0xb8, 0x28, 0x51, 0x2a, + 0x92, 0xe0, 0x06, 0x36, 0x55, 0x00, 0x6c, 0x99, 0x31, 0xa7, 0x56, 0xb3, + 0x7b, 0x15, 0xcd, 0xc1, 0x32, 0x3a, 0xc0, 0x37, 0x1f, 0xea, 0x29, 0xb6, + 0x75, 0xdf, 0x8a, 0x17, 0x09, 0x45, 0xc2, 0x6e, 0xe2, 0x4c, 0xa5, 0x93, + 0x9b, 0x17, 0x08, 0x27, 0x75, 0x33, 0xdb, 0x1f, 0xab, 0x37, 0xad, 0x8e, + 0xaa, 0xef, 0x0b, 0x82, 0xaa, 0xa7, 0xae, 0x2c, 0x43, 0x4d, 0x8f, 0xa0, + 0x43, 0xd7, 0xa1, 0x34, 0xeb, 0xc0, 0x4e, 0xbd, 0x64, 0xfc, 0xc8, 0x6a, + 0x56, 0xa8, 0xfc, 0x9e, 0x2d, 0x5f, 0x7a, 0xa3, 0x72, 0x06, 0x79, 0x38, + 0x33, 0x05, 0xa7, 0xf0, 0x09, 0x48, 0x55, 0xfe, 0x3f, 0xab, 0x25, 0x8e, + 0x76, 0x1d, 0x12, 0x5a, 0x20, 0x68, 0xfb, 0x51, 0x51, 0x33, 0x40, 0x37, + 0x0c, 0x90, 0x98, 0x6f, 0x66, 0x3f, 0x40, 0xa2, 0x2e, 0x3c, 0xd1, 0x22, + 0x51, 0x54, 0x25, 0x7e, 0x4c, 0x5d, 0x96, 0xb2, 0x65, 0x0f, 0xa3, 0xdf, + 0x8e, 0x97, 0xfe, 0xeb, 0xe7, 0xc6, 0x22, 0x2a, 0x47, 0x3a, 0x78, 0x1b, + 0x39, 0x2e, 0xd6, 0xbc, 0x35, 0xb4, 0xf4, 0xc3, 0xf2, 0x6a, 0x12, 0xc9, + 0xe7, 0x6c, 0x9a, 0xfc, 0xed, 0xbc, 0x11, 0xc7, 0x71, 0x09, 0x8f, 0x56, + 0xc1, 0xd8, 0xb6, 0x92, 0x35, 0x97, 0x8e, 0x71, 0xd2, 0xbb, 0xb4, 0xed, + 0xf0, 0x7e, 0xff, 0x58, 0xd9, 0x95, 0x26, 0xea, 0xa9, 0x4d, 0x38, 0x8d, + 0x4e, 0x8e, 0x53, 0xae, 0x7e, 0xe6, 0xe6, 0x82, 0x35, 0x96, 0xab, 0x0f, + 0x04, 0x0f, 0xf2, 0xac, 0x1b, 0xcd, 0x07, 0x17, 0x1b, 0x25, 0x2f, 0x92, + 0xaf, 0x19, 0xa2, 0x1b, 0xa0, 0x7a, 0xc7, 0x4f, 0xb8, 0x1b, 0x89, 0x21, + 0xb5, 0xe2, 0x24, 0xe9, 0x78, 0xae, 0x7d, 0xd7, 0xcc, 0x8e, 0x3f, 0xa7, + 0xe9, 0xbe, 0xe6, 0x79, 0x0f, 0xdf, 0x86, 0xe9, 0xb9, 0xcd, 0x82, 0x7b, + 0xf5, 0x04, 0x89, 0xa0, 0x73, 0x5d, 0xa2, 0x4e, 0xd6, 0xa0, 0x60, 0x21, + 0xe2, 0xfe, 0xd3, 0xf4, 0x19, 0x8b, 0x6a, 0x03, 0x12, 0x9c, 0x51, 0x9a, + 0x41, 0x4e, 0xf6, 0xb4, 0x6e, 0x0c, 0x43, 0xf5, 0x00, 0x00, 0x78, 0x12, + 0xdd, 0x21, 0xa8, 0xc7, 0x21, 0xa1, 0x4e, 0x44, 0x10, 0xd0, 0xdb, 0x6f, + 0x0b, 0x4c, 0xe7, 0x7a, 0x8c, 0x0c, 0xaa, 0xb6, 0x9a, 0x7d, 0xa9, 0xff, + 0x5a, 0x2e, 0x15, 0x9e, 0x6f, 0xea, 0xe1, 0x42, 0x0c, 0x9c, 0x5a, 0x3b, + 0xd5, 0xe6, 0xde, 0x23, 0x3f, 0x9c, 0x45, 0x20, 0x67, 0x96, 0x50, 0x16, + 0x80, 0x42, 0xe7, 0x67, 0x7d, 0x24, 0xdc, 0x00, 0xaa, 0x01, 0x8a, 0xa3, + 0x61, 0xfe, 0x9a, 0xce, 0xc1, 0xe5, 0x2e, 0x19, 0x85, 0x04, 0xe6, 0x7b, + 0xe8, 0x7a, 0xbc, 0x9d, 0xfe, 0x71, 0x29, 0x1d, 0x17, 0xae, 0x6b, 0x1a, + 0x64, 0xd7, 0xfe, 0x18, 0x29, 0x07, 0x9b, 0x49, 0x43, 0xba, 0x29, 0x37, + 0xa8, 0xb0, 0x26, 0x27, 0x6b, 0x7d, 0xde, 0x49, 0x12, 0x90, 0x05, 0xe2, + 0x2c, 0xd8, 0x08, 0xd0, 0x5d, 0x74, 0xa7, 0x15, 0xbe, 0x34, 0x34, 0x6d, + 0xad, 0xfb, 0xa8, 0x01, 0x4a, 0x6c, 0x98, 0xba, 0x84, 0x38, 0xbd, 0x05, + 0xe8, 0x87, 0x27, 0x91, 0x3f, 0xb8, 0xe9, 0x06, 0x27, 0xda, 0x56, 0x07, + 0xaa, 0xea, 0xf4, 0x80, 0x5c, 0x12, 0x44, 0xbe, 0x23, 0xb3, 0x63, 0x9f, + 0x5f, 0x37, 0xa7, 0x53, 0x4c, 0xfc, 0x4d, 0x87, 0xeb, 0x91, 0xe8, 0xd7, + 0x5a, 0xd6, 0xca, 0x67, 0x2d, 0x2f, 0x5a, 0x0e, 0xc7, 0x82, 0x78, 0xa4, + 0xf3, 0x56, 0x07, 0xa5, 0xab, 0x6d, 0x09, 0xd2, 0x0d, 0x08, 0x6b, 0x6e, + 0x1f, 0xc1, 0xf2, 0x91, 0x1a, 0x39, 0xfe, 0x14, 0x56, 0x3f, 0xeb, 0x9f, + 0x14, 0xc2, 0xb3, 0xb2, 0xc2, 0x8d, 0xc2, 0xee, 0x7e, 0xf0, 0x7d, 0x92, + 0xd2, 0xc3, 0x57, 0x3e, 0x2c, 0x07, 0x1b, 0x6a, 0x9b, 0x3b, 0x79, 0x59, + 0xc9, 0x22, 0x96, 0x6c, 0x3e, 0x37, 0xd3, 0x0e, 0x5c, 0xf6, 0x8f, 0xa9, + 0xaa, 0xc9, 0xa4, 0x4b, 0xaf, 0x5d, 0x1a, 0xb6, 0xf3, 0x91, 0x32, 0x4f, + 0xca, 0x72, 0xa0, 0x42, 0x01, 0x51, 0xaf, 0x19, 0x89, 0xc4, 0xcc, 0x9b, + 0xf3, 0x52, 0xe9, 0xa6, 0xf2, 0x71, 0x6f, 0x5a, 0x38, 0x02, 0xb8, 0x75, + 0x88, 0x5f, 0x8d, 0x12, 0xc5, 0x55, 0x4f, 0xd1, 0xba, 0xf2, 0x24, 0xdc, + 0x63, 0x5f, 0x93, 0xc7, 0xf3, 0xe7, 0x59, 0xac, 0xc3, 0xed, 0xbc, 0x02, + 0xe3, 0xad, 0xb2, 0x8e, 0x2c, 0x2d, 0x47, 0xb4, 0x34, 0x8d, 0xae, 0x44, + 0xc8, 0x5f, 0x14, 0xe8, 0x8e, 0x7b, 0xc3, 0x60, 0x53, 0x9a, 0x51, 0xea, + 0x7f, 0x2f, 0xb6, 0x62, 0x61, 0xf7, 0xc0, 0x18, 0x0f, 0x20, 0x79, 0x13, + 0x5c, 0xe8, 0xca, 0x04, 0x29, 0x5f, 0x70, 0x4d, 0x88, 0xa2, 0x43, 0x20, + 0x57, 0x33, 0x04, 0x74, 0x8e, 0x7c, 0x89, 0xd4, 0x56, 0x8f, 0x93, 0x86, + 0x81, 0x6c, 0x11, 0xfc, 0x32, 0x0e, 0xb0, 0x3e, 0xe5, 0x13, 0xbf, 0x76, + 0x62, 0xcc, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x0e, 0xf8, 0x8f, 0xde, 0xfd, 0xfd, 0xcf, 0xd1, 0x6f, 0x9f, 0xf2, 0xb6, + 0xb6, 0x59, 0xb2, 0x73, 0x1c, 0x3c, 0x0d, 0xb0, 0x4d, 0xb8, 0x96, 0xc6, + 0xeb, 0xe5, 0xf8, 0x0d, 0x3e, 0xd7, 0x0c, 0xbd, 0x9c, 0xaa, 0xd5, 0x1c, + 0x19, 0x9a, 0x4c, 0x8e, 0xfa, 0xac, 0x68, 0x74, 0x16, 0x06, 0xb5, 0x49, + 0xe7, 0xd5, 0x6f, 0x4f, 0xcc, 0xd9, 0x02, 0x74, 0xd6, 0x08, 0x73, 0x7c, + 0xa9, 0xfa, 0x3e, 0x50, 0x87, 0xf7, 0xfb, 0xa6, 0x94, 0xdc, 0xb1, 0x40, + 0xec, 0xa7, 0xa9, 0x39, 0xff, 0x40, 0x4a, 0x97, 0x9b, 0xcc, 0x57, 0x66, + 0x68, 0xd6, 0xa8, 0x4d, 0x13, 0x06, 0x0e, 0x03, 0xc4, 0xdf, 0x7a, 0xe4, + 0x2f, 0x0e, 0xd7, 0x54, 0xe0, 0xbd, 0x93, 0xeb, 0x82, 0xd8, 0x05, 0x2d, + 0xa2, 0xf0, 0x4e, 0xd0, 0xf9, 0x3e, 0x3e, 0x6b, 0x3d, 0x08, 0x39, 0x4e, + 0x35, 0x13, 0x7b, 0x3b, 0x39, 0x2c, 0x47, 0x2c, 0x61, 0x9f, 0xfd, 0x59, + 0x88, 0x5f, 0x65, 0x08, 0xa9, 0x66, 0xec, 0xb5, 0x21, 0xf3, 0xe9, 0xba, + 0x11, 0x63, 0x24, 0x6c, 0xf4, 0x50, 0x3a, 0xe5, 0x0c, 0x06, 0x39, 0x69, + 0x2f, 0xca, 0x0f, 0x48, 0xbe, 0x95, 0x7d, 0x13, 0x3d, 0xa5, 0x75, 0x69, + 0x85, 0xc8, 0xb3, 0x72, 0x72, 0x3c, 0x4f, 0x96, 0xe7, 0xb7, 0xbd, 0xe7, + 0x76, 0xba, 0xac, 0xc0, 0x07, 0x4d, 0xc1, 0xed, 0xb9, 0xf0, 0x91, 0x2e, + 0x36, 0xb7, 0x5b, 0x1c, 0xb7, 0xd6, 0xb3, 0x45, 0x7d, 0x0a, 0xf5, 0x43, + 0xdd, 0x7a, 0x8b, 0x4e, 0x18, 0xf2, 0xf3, 0x19, 0xcd, 0x4a, 0xda, 0x3c, + 0x1b, 0x05, 0x27, 0x67, 0x43, 0xa9, 0x8e, 0xe7, 0x4a, 0x95, 0xa9, 0xad, + 0x6c, 0x8c, 0xb2, 0x2e, 0x12, 0xcb, 0xf3, 0xeb, 0x65, 0x26, 0xf4, 0x3e, + 0x86, 0xee, 0x7e, 0xd9, 0xba, 0xce, 0x8d, 0x15, 0x3e, 0xa8, 0x40, 0x59, + 0x1d, 0x27, 0x78, 0x75, 0xf0, 0xf9, 0x33, 0xb5, 0x32, 0xa9, 0x66, 0xe6, + 0x2e, 0x2e, 0x3d, 0xf5, 0x4a, 0xf0, 0x97, 0x2d, 0xe7, 0x43, 0x85, 0x43, + 0x61, 0x25, 0x15, 0x13, 0x9e, 0x8e, 0xf6, 0x78, 0xe8, 0x67, 0xba, 0xc2, + 0x6d, 0xda, 0x46, 0x25, 0x76, 0xd9, 0x9b, 0x69, 0x95, 0x4b, 0x50, 0x8c, + 0xb7, 0x36, 0x49, 0xbc, 0xd7, 0x39, 0x69, 0xb9, 0xc1, 0x5f, 0x5f, 0xcc, + 0x83, 0x4c, 0x16, 0xb8, 0x0c, 0x85, 0xf1, 0xa4, 0x57, 0x6c, 0x22, 0x1f, + 0x60, 0x0c, 0xff, 0xb6, 0xc9, 0xf7, 0x21, 0x2d, 0x35, 0x78, 0x31, 0x79, + 0xd0, 0x6d, 0x61, 0xec, 0x61, 0x04, 0x75, 0x5c, 0x06, 0xc3, 0x53, 0x1b, + 0xb5, 0xdc, 0x23, 0xb9, 0xd9, 0x07, 0xd1, 0xd0, 0xb3, 0xa5, 0xab, 0xd9, + 0xbe, 0xb7, 0xdc, 0xae, 0x3f, 0x3e, 0xd7, 0x2a, 0x79, 0x3f, 0x9c, 0x27, + 0x81, 0x8d, 0x61, 0xe8, 0x46, 0x8f, 0x05, 0xf4, 0x9c, 0x30, 0x35, 0x9a, + 0x2f, 0x62, 0x84, 0x7c, 0xa5, 0x95, 0x68, 0x34, 0xe6, 0xf0, 0xb9, 0x42, + 0xd4, 0x37, 0xc6, 0xd2, 0x35, 0x1f, 0x7b, 0xe0, 0xa6, 0x92, 0xcf, 0xf7, + 0x0f, 0x08, 0x10, 0x79, 0xbd, 0xa8, 0x7c, 0x4e, 0xef, 0xf1, 0x01, 0x8d, + 0x1b, 0x0c, 0x98, 0x46, 0x28, 0xdc, 0xd5, 0xa8, 0xcf, 0x67, 0x7d, 0x87, + 0x2a, 0x8f, 0xdd, 0x52, 0x43, 0x5a, 0x55, 0x80, 0x88, 0xa6, 0xcd, 0x9c, + 0x5d, 0x36, 0xae, 0xef, 0x61, 0x43, 0xec, 0xf0, 0x7f, 0x92, 0x21, 0x1f, + 0xa2, 0xa3, 0x76, 0x0e, 0x5d, 0xf3, 0xa7, 0xe7, 0x7d, 0xb0, 0x2c, 0x94, + 0x36, 0x95, 0x34, 0x4e, 0x04, 0xfb, 0x51, 0xf9, 0xe6, 0x7e, 0x56, 0x7a, + 0x59, 0xce, 0x0a, 0x45, 0x7e, 0xeb, 0xc4, 0xbc, 0xfd, 0x20, 0xaa, 0x34, + 0x6b, 0xee, 0x3b, 0x09, 0xe8, 0x00, 0x4b, 0xfc, 0x68, 0x24, 0x43, 0xdb, + 0x09, 0x58, 0xd0, 0xb6, 0xbf, 0xaf, 0x1d, 0x7f, 0x8a, 0x4c, 0x9e, 0x51, + 0x97, 0x97, 0xe1, 0x0c, 0x0d, 0xaf, 0xd1, 0x1e, 0x62, 0xad, 0x70, 0xa5, + 0x8a, 0x24, 0x2f, 0x4a, 0xa6, 0x55, 0xb1, 0x44, 0x09, 0x88, 0xab, 0xa5, + 0x45, 0x28, 0xa0, 0x34, 0x9e, 0x14, 0x2c, 0xf9, 0x0f, 0xb8, 0x33, 0x8f, + 0xcc, 0xba, 0x50, 0x34, 0x4c, 0x96, 0x89, 0x09, 0xb9, 0xa8, 0xfb, 0xac, + 0x59, 0x73, 0xea, 0x61, 0xbc, 0x0d, 0x24, 0x3a, 0x20, 0xc2, 0x76, 0xfc, + 0x2e, 0xce, 0xfb, 0x75, 0x00, 0xca, 0x58, 0xbd, 0xab, 0x61, 0x9b, 0x13, + 0x2b, 0xa3, 0xf6, 0x15, 0x55, 0x83, 0x23, 0xc4, 0xf3, 0x4c, 0x89, 0xc5, + 0x4a, 0x18, 0x5c, 0x8d, 0x41, 0xcc, 0x06, 0x7b, 0xe3, 0x2a, 0x1f, 0x6a, + 0x57, 0xbc, 0x54, 0x61, 0x0c, 0xf2, 0xec, 0xbf, 0xb0, 0xf0, 0x21, 0xde, + 0xfc, 0xe4, 0xef, 0xce, 0x47, 0xc8, 0xdc, 0x11, 0xc7, 0x8a, 0x12, 0x97, + 0x68, 0x1d, 0x9e, 0x9a, 0xbf, 0xad, 0x62, 0x7e, 0x4b, 0x88, 0xd7, 0x20, + 0x22, 0xce, 0x5e, 0xe3, 0x87, 0x12, 0xa3, 0x05, 0xef, 0x1f, 0x05, 0xb1, + 0xbd, 0x1b, 0x80, 0x43, 0x84, 0x33, 0x8b, 0x87, 0xa5, 0xc2, 0xe1, 0x49, + 0xa8, 0x75, 0x49, 0x9b, 0x1b, 0x64, 0x8a, 0xd0, 0x86, 0x10, 0xa8, 0x72, + 0xeb, 0x2e, 0xe7, 0x3f, 0xaa, 0x6b, 0x4a, 0x22, 0xae, 0x17, 0x8f, 0x10, + 0x22, 0x03, 0x66, 0x67, 0x35, 0x40, 0x29, 0x1e, 0xf2, 0x05, 0x36, 0xd5, + 0xed, 0xe2, 0x2a, 0xcc, 0x77, 0xe2, 0x16, 0xef, 0xa7, 0x9b, 0xe1, 0x1b, + 0xba, 0xf3, 0xf5, 0x74, 0x6c, 0x2a, 0x98, 0x8a, 0x14, 0xaf, 0x2c, 0xab, + 0xfb, 0x51, 0x53, 0x75, 0x17, 0xcb, 0x5c, 0x86, 0xb5, 0x60, 0x70, 0x29, + 0x65, 0x69, 0x49, 0x42, 0x4f, 0x42, 0x6b, 0xc7, 0xdb, 0x98, 0x7d, 0x1e, + 0xf8, 0x45, 0xb2, 0x33, 0xd6, 0x34, 0x26, 0xa6, 0x7f, 0x76, 0x31, 0x13, + 0x13, 0x9d, 0xd2, 0xb0, 0x30, 0x0b, 0x0b, 0x3e, 0x1a, 0x84, 0xb0, 0xbd, + 0x81, 0x34, 0x25, 0x73, 0x99, 0x87, 0x1a, 0xc8, 0x44, 0x34, 0x9d, 0x1a, + 0x3d, 0x76, 0x44, 0x1d, 0xe2, 0x22, 0xad, 0x3d, 0xb2, 0xa3, 0x1c, 0xd5, + 0x27, 0x8c, 0xc6, 0x84, 0xdf, 0x33, 0xbe, 0xb2, 0xa7, 0xb9, 0xc5, 0x6e, + 0x48, 0xdc, 0xe9, 0xf8, 0xef, 0xfc, 0xaa, 0x1f, 0x5e, 0x41, 0x48, 0x1e, + 0xe0, 0xb9, 0xd6, 0x6e, 0x7a, 0x9c, 0xa3, 0x98, 0x4b, 0xfa, 0x90, 0xa4, + 0x58, 0x33, 0x85, 0x3b, 0x11, 0x44, 0x83, 0x4b, 0x1e, 0x0e, 0x5d, 0x11, + 0x36, 0x15, 0xe1, 0xbf, 0x15, 0x04, 0x8e, 0x88, 0xc6, 0x18, 0x53, 0xc3, + 0x8d, 0x28, 0x86, 0x25, 0xef, 0x55, 0x7b, 0xf6, 0x85, 0xf8, 0xed, 0x3b, + 0xcf, 0x5d, 0xa6, 0xc7, 0x66, 0xb7, 0xbe, 0x14, 0xf0, 0x62, 0x89, 0x1f, + 0x32, 0x1e, 0x86, 0x2a, 0x93, 0xd5, 0xca, 0x37, 0x03, 0x0b, 0xf8, 0x0f, + 0xca, 0x50, 0x6c, 0x16, 0x2b, 0xf0, 0x77, 0xca, 0xbb, 0x8e, 0x95, 0x11, + 0xef, 0x5b, 0xbe, 0x2f, 0x62, 0x50, 0xb8, 0x3d, 0xff, 0xfa, 0x30, 0x21, + 0xb2, 0x86, 0x3f, 0x50, 0x57, 0x98, 0x79, 0x15, 0xce, 0x3e, 0xbf, 0x49, + 0x58, 0xb0, 0xb5, 0xd7, 0xbe, 0x01, 0x55, 0xee, 0x60, 0x14, 0x9d, 0x5b, + 0x57, 0x48, 0x05, 0x72, 0x6a, 0x23, 0x29, 0xeb, 0xf3, 0x36, 0x2a, 0xc1, + 0xda, 0x5e, 0x4a, 0x63, 0xc4, 0x6b, 0x04, 0xe8, 0xe8, 0xc1, 0xb5, 0xc4, + 0x2d, 0x60, 0x1f, 0xa0, 0x2b, 0x33, 0xa5, 0xb7, 0x82, 0x59, 0x21, 0xba, + 0x13, 0xda, 0x79, 0xda, 0x5a, 0xb1, 0x82, 0x5b, 0x52, 0x7f, 0x0c, 0x70, + 0x75, 0x65, 0xe0, 0x44, 0xb3, 0xca, 0xd0, 0x09, 0x38, 0x24, 0x83, 0x8e, + 0x0c, 0x4c, 0xef, 0x96, 0xe4, 0x04, 0x30, 0x46, 0x23, 0x6a, 0x28, 0x13, + 0x1d, 0x37, 0x14, 0x75, 0x6e, 0xd0, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x21, 0xa2, 0xf0, 0x7d, 0x29, 0x8f, 0x62, 0x2e, + 0xf4, 0x0e, 0x14, 0x9b, 0x60, 0x38, 0xc0, 0x95, 0xfb, 0x3c, 0x90, 0x5a, + 0xa0, 0x1f, 0x30, 0x09, 0xfc, 0x6d, 0xa9, 0xd1, 0x7b, 0x0b, 0x7c, 0x78, + 0xf9, 0xf6, 0xa8, 0x5e, 0xa6, 0x7a, 0xf6, 0x1c, 0xab, 0x1b, 0x0e, 0xa9, + 0x08, 0xfd, 0xd9, 0x97, 0x08, 0x24, 0x2b, 0xda, 0x08, 0x8b, 0x0c, 0x07, + 0x70, 0x15, 0xa8, 0x0c, 0x86, 0xfc, 0xd1, 0x84, 0xba, 0xd0, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x35, 0x7a, 0xab, 0xaa, + 0xbe, 0xd7, 0xad, 0x22, 0x99, 0x46, 0xbb, 0x78, 0xfd, 0x47, 0x8f, 0x2a, + 0x4a, 0xa6, 0x2f, 0x8d, 0x15, 0x07, 0xed, 0x26, 0x1d, 0xb3, 0x12, 0xd3, + 0x88, 0x0f, 0xf1, 0x75, 0x2a, 0x07, 0x62, 0xac, 0xbf, 0x52, 0x4a, 0xc3, + 0x12, 0xe5, 0x3c, 0xea, 0xa6, 0x1e, 0x57, 0x90, 0x56, 0x60, 0x7d, 0xcf, + 0x4b, 0x65, 0xaf, 0xee, 0x17, 0x56, 0xbe, 0xd2, 0x38, 0x3f, 0xd6, 0xbc, + 0xef, 0xa7, 0x32, 0xb7, 0x10, 0xe9, 0xbd, 0x97, 0x45, 0x92, 0x3c, 0xd3, + 0x35, 0x2e, 0x59, 0x37, 0x65, 0x5c, 0x7f, 0xd0, 0x99, 0x9c, 0x01, 0xe9, + 0x1f, 0x65, 0xe9, 0xec, 0x0f, 0x2d, 0x46, 0xbc, 0xd4, 0x8f, 0x51, 0x1c, + 0xa0, 0xa4, 0x9b, 0x4f, 0x95, 0x54, 0xb0, 0x50, 0x74, 0xfa, 0x0f, 0xe6, + 0x55, 0x81, 0xce, 0x0f, 0xd1, 0x25, 0x56, 0xc8, 0x2f, 0x3a, 0x65, 0xd4, + 0x86, 0x4a, 0x8e, 0xff, 0x5a, 0xcc, 0x67, 0x96, 0xcc, 0x65, 0x0d, 0x20, + 0xee, 0xba, 0x6b, 0xcb, 0xde, 0x10, 0x2f, 0xbf, 0x67, 0x6d, 0xbe, 0xef, + 0x72, 0xfc, 0x25, 0x62, 0xbf, 0xbb, 0xc5, 0xe0, 0x7b, 0x4c, 0x32, 0xc5, + 0xdb, 0x9f, 0xb5, 0xe2, 0x75, 0x8a, 0xba, 0xbb, 0x69, 0x28, 0xb6, 0x41, + 0x25, 0x83, 0x67, 0x35, 0x1b, 0xd7, 0xb3, 0xd7, 0x58, 0x54, 0x8a, 0x0b, + 0x7c, 0xf3, 0x05, 0xcf, 0x2c, 0x78, 0x70, 0xc6, 0xed, 0x7e, 0x56, 0xb6, + 0x4e, 0x48, 0xaa, 0x57, 0xc4, 0xb0, 0xb2, 0xa0, 0xca, 0x50, 0xe1, 0xc7, + 0x41, 0xea, 0xac, 0x5f, 0x18, 0x13, 0xe5, 0x85, 0x78, 0x3f, 0x05, 0xf3, + 0xfd, 0x74, 0x7a, 0x42, 0x61, 0x91, 0x19, 0xc6, 0x19, 0xe9, 0xd2, 0x78, + 0x2c, 0xb1, 0xa3, 0x7f, 0x62, 0xea, 0x2a, 0x35, 0x1c, 0x55, 0xa3, 0xf7, + 0xdc, 0xec, 0x48, 0x23, 0x99, 0x8d, 0xe1, 0x4d, 0x45, 0xad, 0x92, 0xc6, + 0xf4, 0xa2, 0xe5, 0xe6, 0x58, 0xe4, 0xd5, 0x37, 0xd0, 0x47, 0x0b, 0x64, + 0x68, 0x48, 0x7e, 0xeb, 0xbe, 0x5e, 0x74, 0xd1, 0xc4, 0xa5, 0x60, 0xd0, + 0x30, 0x62, 0xbc, 0x81, 0xc4, 0x01, 0x68, 0x18, 0xf3, 0xac, 0x9d, 0xb1, + 0x4d, 0xdd, 0x8b, 0xd2, 0x54, 0x5d, 0xd1, 0x1c, 0xee, 0x75, 0x9e, 0x99, + 0x42, 0x69, 0x38, 0xcc, 0x66, 0x24, 0xd9, 0x8f, 0x70, 0x98, 0xc3, 0x5e, + 0x08, 0xf0, 0xd8, 0x2d, 0xe6, 0x52, 0x48, 0xdf, 0xd0, 0x03, 0x04, 0x92, + 0xab, 0xa1, 0xa1, 0x2f, 0x7d, 0x84, 0xb2, 0x82, 0x51, 0x56, 0x74, 0x4a, + 0x94, 0xff, 0xd2, 0xe4, 0x4e, 0x1a, 0xbd, 0x18, 0xab, 0x33, 0x68, 0x0e, + 0x4f, 0x99, 0x1d, 0x7e, 0x02, 0x3f, 0x1f, 0x50, 0x05, 0xf8, 0x59, 0x47, + 0x97, 0x98, 0x60, 0xb1, 0x30, 0xb1, 0x14, 0xac, 0x2c, 0x0a, 0xa8, 0x97, + 0x83, 0xf5, 0x5a, 0x5c, 0x87, 0xe5, 0x36, 0x26, 0xec, 0xb4, 0x94, 0x46, + 0x9a, 0xad, 0x2b, 0x9a, 0xb7, 0xac, 0xc4, 0x1a, 0x55, 0x53, 0xc0, 0x16, + 0x91, 0x1c, 0xd6, 0xaa, 0x6b, 0xdd, 0x85, 0x6a, 0x54, 0xec, 0x7c, 0xa1, + 0xd5, 0x18, 0x00, 0x74, 0xd2, 0xf1, 0x7e, 0xad, 0x7c, 0xa8, 0x85, 0x9b, + 0xc0, 0x9f, 0x4f, 0x3b, 0xd9, 0x08, 0xc8, 0x9d, 0x31, 0x22, 0x7a, 0x53, + 0xa8, 0xbd, 0x00, 0xdf, 0xe8, 0x39, 0x52, 0xe9, 0x14, 0x74, 0x7b, 0x53, + 0xf9, 0xbd, 0x29, 0x8e, 0x5d, 0xf2, 0x35, 0x3b, 0xe3, 0x48, 0xbf, 0xa0, + 0xc4, 0x3d, 0x40, 0xb4, 0xf2, 0x7c, 0xd0, 0xe3, 0x17, 0x11, 0x5b, 0xd6, + 0x55, 0xd2, 0x54, 0xcf, 0x20, 0x8d, 0x74, 0x4a, 0x6b, 0xe9, 0x5d, 0xfe, + 0x72, 0x14, 0x6a, 0x11, 0x8b, 0x14, 0x19, 0xba, 0x63, 0xe4, 0x6b, 0x39, + 0xb4, 0x90, 0x67, 0x79, 0x56, 0x31, 0xd3, 0xb5, 0xeb, 0x9e, 0x95, 0x4b, + 0x1e, 0x04, 0x20, 0xd8, 0xbe, 0xe8, 0x1c, 0xd7, 0x95, 0xcb, 0x57, 0x60, + 0xe6, 0x11, 0x35, 0x42, 0x90, 0xfd, 0xb2, 0xe4, 0x9b, 0x24, 0x70, 0xc0, + 0xc3, 0xa9, 0x8a, 0xc9, 0x46, 0xd0, 0xea, 0xc9, 0x93, 0x7d, 0x9f, 0x64, + 0x12, 0x54, 0x09, 0xb7, 0xc2, 0x4d, 0x6e, 0xcc, 0x60, 0x07, 0x36, 0x31, + 0x64, 0x3d, 0x1e, 0xd3, 0x86, 0x47, 0x47, 0x42, 0x76, 0xb6, 0xf0, 0xe5, + 0xb4, 0xe7, 0xbe, 0x47, 0x91, 0x78, 0xbe, 0x06, 0xf1, 0x6e, 0x58, 0xce, + 0x32, 0x13, 0x26, 0x34, 0x92, 0xae, 0xb2, 0x29, 0xd0, 0x30, 0x55, 0xfd, + 0x89, 0x6a, 0xbf, 0x3e, 0xdf, 0x11, 0x39, 0xe4, 0xfd, 0x56, 0xd7, 0x2f, + 0x89, 0x96, 0x08, 0x54, 0xaa, 0xab, 0x8b, 0xfa, 0x65, 0xe5, 0x64, 0xff, + 0x24, 0x25, 0x8f, 0x7d, 0xf6, 0xb1, 0x7f, 0x2f, 0xa6, 0xf6, 0x46, 0xab, + 0x61, 0xfd, 0x47, 0xad, 0x6d, 0x38, 0x6d, 0xc1, 0xe9, 0x4a, 0xf1, 0x85, + 0x05, 0x0e, 0x69, 0x48, 0x7c, 0xa6, 0x76, 0x61, 0xe3, 0x94, 0xf2, 0xd6, + 0x7a, 0x9c, 0x79, 0xc0, 0x2a, 0x51, 0x23, 0xc6, 0xaf, 0x29, 0x04, 0x0f, + 0x47, 0xc2, 0x93, 0xd7, 0x64, 0xe5, 0x37, 0x2e, 0x53, 0x3b, 0xb7, 0x7c, + 0x9c, 0xb4, 0x63, 0x13, 0xc7, 0x56, 0x90, 0xe9, 0x53, 0xd5, 0x86, 0x2b, + 0x96, 0x41, 0x42, 0x56, 0xc5, 0x16, 0xd7, 0x9e, 0x30, 0xce, 0xa1, 0x0d, + 0x93, 0x5d, 0x11, 0x07, 0xb2, 0x95, 0xfd, 0xf6, 0x0b, 0x28, 0x95, 0x1a, + 0x8f, 0xfa, 0xe1, 0x57, 0x7e, 0x06, 0xff, 0x18, 0xaf, 0xe3, 0x4f, 0x3c, + 0x34, 0x5b, 0xd4, 0x46, 0x1a, 0xd1, 0xd1, 0x7e, 0x55, 0xba, 0x5d, 0x2a, + 0x1f, 0x42, 0x49, 0x95, 0x75, 0x5f, 0x80, 0x60, 0x02, 0x01, 0xdb, 0x36, + 0xad, 0x68, 0x69, 0x1e, 0x0b, 0x90, 0x3f, 0xa6, 0xb6, 0x2f, 0x66, 0xa6, + 0x7d, 0x81, 0x8c, 0xa0, 0xee, 0x05, 0x95, 0xbc, 0xb3, 0x7c, 0x18, 0xd4, + 0x1b, 0x40, 0x96, 0xf5, 0x05, 0x9d, 0x27, 0x3b, 0x78, 0xfc, 0x19, 0x18, + 0xc0, 0x61, 0xa0, 0xd6, 0xf9, 0xc0, 0x3f, 0xe5, 0x48, 0x35, 0x0f, 0x8b, + 0x0d, 0xfb, 0x31, 0xb7, 0x32, 0x40, 0x1d, 0x69, 0x12, 0x5a, 0x23, 0xf0, + 0xce, 0xe9, 0x5e, 0xa6, 0x68, 0x6b, 0xe1, 0xe2, 0x68, 0x07, 0x02, 0x0d, + 0x7a, 0xc2, 0x0a, 0x40, 0x10, 0x5e, 0x94, 0xba, 0x77, 0x1d, 0xf7, 0xac, + 0xec, 0x79, 0xa9, 0xa1, 0x8a, 0xb8, 0x49, 0x32, 0x08, 0xe0, 0x18, 0xa8, + 0x3d, 0x69, 0x41, 0x5d, 0x30, 0x3b, 0xb6, 0x91, 0x46, 0x8d, 0x81, 0x10, + 0xb0, 0xc2, 0xed, 0xa0, 0x4e, 0x59, 0x48, 0xd8, 0x64, 0x7d, 0x2d, 0x46, + 0xf2, 0x8a, 0x2e, 0x5d, 0x0c, 0x4d, 0x9f, 0xfe, 0x7b, 0x5e, 0xbf, 0x1a, + 0x78, 0xdf, 0xfc, 0x0f, 0x04, 0x37, 0x72, 0x1a, 0x09, 0xb8, 0x6e, 0x1b, + 0xf1, 0x18, 0x7d, 0x83, 0x44, 0xaa, 0x9b, 0x71, 0xe1, 0x03, 0x04, 0x83, + 0xe5, 0xaa, 0xc0, 0xd4, 0xa7, 0x80, 0x10, 0x35, 0x09, 0xae, 0xf7, 0xe1, + 0x5e, 0x7c, 0x31, 0x20, 0x43, 0x82, 0xda, 0x07, 0x39, 0xfe, 0x8f, 0x9d, + 0x70, 0x3c, 0x57, 0x43, 0x01, 0x51, 0x37, 0x2e, 0x97, 0xef, 0xcf, 0x05, + 0x44, 0x75, 0x69, 0xf7, 0xdb, 0xda, 0x80, 0x78, 0x0c, 0xcc, 0xc1, 0x49, + 0xac, 0x3b, 0x7e, 0x27, 0x6a, 0xbb, 0xdf, 0x45, 0x5b, 0x3b, 0x29, 0xf6, + 0x1b, 0xa9, 0x25, 0xf9, 0x2f, 0xcf, 0x37, 0x71, 0x33, 0xb4, 0x90, 0xd7, + 0x9b, 0x87, 0x41, 0x15, 0xd1, 0xa6, 0x39, 0xa7, 0xa9, 0xcd, 0x66, 0x29, + 0x59, 0xb4, 0x53, 0x12, 0xa1, 0x20, 0xd5, 0x04, 0xca, 0x40, 0x31, 0xfa, + 0x6f, 0xbb, 0x92, 0x04, 0xf3, 0xc2, 0x10, 0x0d, 0xc1, 0x19, 0x78, 0x8c, + 0x82, 0xed, 0x92, 0x3a, 0x6b, 0xd1, 0x3d, 0xe8, 0xac, 0x55, 0xe4, 0x8c, + 0xc6, 0xd4, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0xc2, 0x1d, 0x86, 0xe4, 0xf6, 0xa1, 0xbe, 0xf5, 0xf3, 0x36, 0x9d, 0x32, + 0x80, 0x17, 0x3b, 0x1f, 0x18, 0x21, 0xed, 0xa7, 0xf5, 0xaf, 0xf1, 0x94, + 0xe2, 0xa7, 0x08, 0xd5, 0xca, 0x18, 0x45, 0xf5, 0x68, 0x94, 0x82, 0x61, + 0xf7, 0xb7, 0xb2, 0xfa, 0xd4, 0x5e, 0x32, 0xd0, 0xf0, 0x20, 0x66, 0x83, + 0xd1, 0x6b, 0x3c, 0xdf, 0x73, 0xeb, 0x73, 0x82, 0x09, 0x9b, 0xd0, 0xc5, + 0xb0, 0x9f, 0x01, 0x77, 0x85, 0xcc, 0x6e, 0x23, 0xb7, 0x00, 0x45, 0xe0, + 0xa6, 0x01, 0x29, 0x1d, 0x8b, 0xc4, 0xe0, 0xc2, 0xe0, 0x4f, 0x3b, 0x07, + 0xd5, 0xac, 0x6b, 0x88, 0xb8, 0xa4, 0xe2, 0x5c, 0x19, 0xe9, 0x98, 0x72, + 0xa5, 0x6b, 0xf5, 0xa4, 0xf7, 0x15, 0xaf, 0xfb, 0xb4, 0x80, 0x9a, 0xe3, + 0xa5, 0x35, 0x2f, 0x45, 0x81, 0xf1, 0x8b, 0x2d, 0x26, 0x5c, 0x65, 0xa9, + 0x5b, 0x6e, 0x83, 0xc3, 0x62, 0x2f, 0x84, 0xef, 0x11, 0xa5, 0x58, 0x48, + 0xe9, 0x67, 0x7e, 0xd3, 0x0b, 0x5d, 0x51, 0x80, 0x39, 0x08, 0x8e, 0xc1, + 0x0d, 0x04, 0x11, 0x5f, 0x72, 0x64, 0x1f, 0x83, 0xf8, 0xd3, 0x09, 0x38, + 0xb6, 0x7f, 0x50, 0x78, 0x27, 0x20, 0xe5, 0xbd, 0x16, 0xbf, 0x51, 0xd8, + 0x4f, 0x67, 0x60, 0xf6, 0x9e, 0xff, 0x08, 0xfe, 0xc6, 0x96, 0xd6, 0x64, + 0x94, 0x28, 0xc6, 0x9a, 0x09, 0x1a, 0x34, 0x08, 0x31, 0x4b, 0x0b, 0x97, + 0x5a, 0x18, 0x72, 0x49, 0xe9, 0x1d, 0xbb, 0x9c, 0xed, 0x7e, 0xb5, 0xc5, + 0xa7, 0xf4, 0x25, 0x7a, 0x26, 0xe9, 0x15, 0x61, 0x85, 0x32, 0xc9, 0xb3, + 0xcf, 0x95, 0xbf, 0x35, 0x10, 0x2d, 0x71, 0xfe, 0x03, 0xd6, 0x69, 0x75, + 0x8d, 0xb7, 0x16, 0xa7, 0x3d, 0x0e, 0xb7, 0x55, 0x6d, 0xa7, 0x9f, 0x10, + 0x7e, 0x7e, 0xff, 0x39, 0xee, 0x8e, 0xa7, 0x81, 0x7d, 0x11, 0xea, 0xa9, + 0xd6, 0xed, 0x54, 0xf8, 0xd2, 0xd5, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0xf9, 0xde, 0x41, 0xe7, 0xa6, 0x88, 0x53, 0x76, + 0x5a, 0x26, 0xc3, 0x5c, 0xf2, 0x58, 0x68, 0x9c, 0xc7, 0x4e, 0x53, 0x18, + 0x53, 0x67, 0x39, 0x23, 0x96, 0xb0, 0xef, 0x58, 0x29, 0xe1, 0x68, 0xd8, + 0xce, 0xc0, 0x41, 0xc2, 0x35, 0x5f, 0x74, 0xfa, 0xdf, 0xc7, 0x0f, 0x80, + 0x50, 0xd1, 0xf6, 0x5a, 0x3a, 0x81, 0xe0, 0xd9, 0x9b, 0x47, 0x96, 0xcd, + 0xc5, 0x0f, 0x91, 0x12, 0x81, 0x77, 0x1e, 0xef, 0x2e, 0xba, 0x16, 0x51, + 0x70, 0x78, 0xdc, 0xa3, 0x84, 0x12, 0x7c, 0x9e, 0x21, 0x7d, 0xa3, 0x5f, + 0xce, 0xa1, 0x25, 0x84, 0x99, 0xa4, 0x2d, 0xa6, 0x0f, 0x95, 0xef, 0xef, + 0x31, 0xe6, 0xf2, 0x18, 0x08, 0x47, 0xd2, 0x5a, 0x39, 0x01, 0x7a, 0xca, + 0xd3, 0x03, 0xb1, 0xc2, 0x48, 0xf4, 0x1f, 0x6d, 0xc2, 0x8c, 0x5c, 0xda, + 0xf5, 0x10, 0xed, 0xfc, 0x2e, 0x0c, 0xb3, 0x52, 0xaa, 0xa9, 0xed, 0xbc, + 0x41, 0xcc, 0xd4, 0x4b, 0x1c, 0xd0, 0xa3, 0x1d, 0xf4, 0xe7, 0x48, 0x34, + 0x4e, 0xcf, 0x3b, 0xb3, 0x71, 0x06, 0xbe, 0x0c, 0x35, 0xbb, 0xb4, 0x17, + 0xd8, 0x8b, 0xba, 0xdd, 0x32, 0x30, 0x51, 0xb1, 0xb1, 0xd6, 0x3a, 0xdc, + 0x3b, 0x25, 0x9a, 0x57, 0xc7, 0x4d, 0xd3, 0x75, 0x93, 0x59, 0x3e, 0x9b, + 0x10, 0xcf, 0xdb, 0x38, 0x75, 0x51, 0xb2, 0x2a, 0x48, 0x78, 0xfc, 0xaa, + 0xe3, 0x91, 0xe7, 0x93, 0xe7, 0x0a, 0x07, 0x2c, 0xf8, 0x88, 0x93, 0xde, + 0x2f, 0xba, 0x7b, 0x72, 0xcd, 0x92, 0xdd, 0xb1, 0xac, 0x1e, 0xe4, 0xe3, + 0x5d, 0xa4, 0x7f, 0x86, 0xa7, 0xcb, 0xb5, 0x81, 0x86, 0xf1, 0xf5, 0xad, + 0xd6, 0x36, 0x08, 0x09, 0x9f, 0x75, 0x6f, 0x4a, 0x5b, 0x30, 0xf8, 0xaf, + 0xd2, 0xbc, 0xb5, 0xbe, 0xf2, 0xeb, 0x9b, 0xbc, 0x11, 0xd4, 0x0c, 0x14, + 0xa6, 0x6f, 0x43, 0xd3, 0xc9, 0x4e, 0xca, 0x9b, 0x4e, 0x46, 0x60, 0x4c, + 0x63, 0xcc, 0x07, 0x36, 0x8c, 0xf2, 0xd1, 0x93, 0x7a, 0x51, 0x49, 0x15, + 0xbf, 0xbf, 0x9e, 0x82, 0x21, 0x06, 0xa0, 0x39, 0x11, 0x1d, 0x6c, 0x41, + 0x72, 0xcd, 0x2a, 0x8a, 0x4a, 0xd0, 0x13, 0x6c, 0x56, 0xf4, 0x00, 0x48, + 0xaf, 0xab, 0xdf, 0xa9, 0xe9, 0xa6, 0xaa, 0x06, 0x61, 0x79, 0xc4, 0x57, + 0x42, 0xca, 0x12, 0x18, 0xcf, 0x81, 0xec, 0x79, 0x19, 0xd2, 0xd2, 0xe3, + 0x1d, 0xc6, 0x6c, 0xd0, 0xd6, 0x0a, 0xfb, 0x70, 0x42, 0x28, 0x25, 0x23, + 0xb6, 0x23, 0x15, 0x28, 0x5e, 0x9f, 0x49, 0xf2, 0x7b, 0x69, 0x74, 0xa5, + 0xb9, 0x26, 0x81, 0xfe, 0x39, 0x3e, 0x3f, 0xc8, 0x7e, 0x9e, 0x5e, 0x8e, + 0xf2, 0xdb, 0x6b, 0xfd, 0xe1, 0xc3, 0x01, 0x4a, 0xba, 0x8f, 0x33, 0x71, + 0x09, 0x80, 0x5d, 0x9c, 0x58, 0x64, 0xb7, 0x90, 0x13, 0x2a, 0xe9, 0x1d, + 0x07, 0x2c, 0x06, 0x70, 0x43, 0x0d, 0xb6, 0x57, 0x02, 0x3c, 0xbe, 0x3c, + 0x42, 0xab, 0x77, 0x15, 0x0e, 0x98, 0xfb, 0xf2, 0x1d, 0x14, 0xd9, 0xb8, + 0xd1, 0x59, 0x2a, 0x67, 0x6f, 0xfc, 0x59, 0x39, 0x33, 0xe0, 0x49, 0x0b, + 0x4e, 0x65, 0x81, 0x9f, 0x71, 0xf2, 0xa5, 0x90, 0x4f, 0x24, 0xc7, 0x05, + 0xfb, 0x77, 0x1e, 0x14, 0xca, 0x2f, 0xfc, 0xac, 0xec, 0xbf, 0xa2, 0x69, + 0x15, 0x0a, 0x6b, 0xa9, 0xa0, 0x74, 0xee, 0xad, 0xa9, 0x50, 0x4d, 0x4d, + 0xab, 0x6e, 0xc1, 0xb3, 0xda, 0xbb, 0xbd, 0xab, 0x00, 0x05, 0x14, 0xc1, + 0xc4, 0x53, 0x7b, 0x78, 0x97, 0x68, 0x3c, 0x05, 0xf2, 0xed, 0x87, 0xca, + 0x86, 0xd1, 0xdf, 0xda, 0xb3, 0x2f, 0x17, 0x87, 0x87, 0x2f, 0xd8, 0xe9, + 0xb2, 0x96, 0xdc, 0x7f, 0x22, 0xf1, 0x2a, 0x9f, 0xfe, 0x54, 0x55, 0xa1, + 0x96, 0xab, 0x9f, 0x61, 0x74, 0xcd, 0x4d, 0x77, 0x38, 0x02, 0x23, 0x29, + 0x28, 0x5b, 0xfc, 0x86, 0x17, 0x40, 0xd4, 0x42, 0x2a, 0x9b, 0x84, 0xf7, + 0x67, 0x2b, 0x3a, 0xc1, 0x31, 0x89, 0x4b, 0x67, 0xd1, 0x7d, 0x6b, 0x36, + 0xec, 0x69, 0x6b, 0x24, 0xca, 0xd6, 0x2d, 0xbb, 0x21, 0xc8, 0x0c, 0x53, + 0x41, 0x29, 0x0b, 0xc1, 0xfe, 0xd5, 0xa3, 0x4c, 0x66, 0x2f, 0xc7, 0xf1, + 0xa8, 0xc0, 0x3d, 0x9a, 0xb9, 0x09, 0x50, 0x3f, 0x09, 0x87, 0xa4, 0x3f, + 0x7a, 0x33, 0xef, 0xf0, 0xfb, 0x77, 0x02, 0x7d, 0x92, 0xaf, 0x73, 0xaa, + 0xcc, 0x3f, 0x66, 0x56, 0xd0, 0x21, 0xd1, 0xe8, 0x0e, 0x47, 0x03, 0x5e, + 0x3b, 0xe9, 0xa2, 0xe3, 0x83, 0x0b, 0x73, 0xd3, 0xaa, 0x94, 0x80, 0xef, + 0x7c, 0xdf, 0xde, 0x86, 0xc3, 0xa9, 0x62, 0x34, 0x76, 0xee, 0x4d, 0x15, + 0x73, 0x7b, 0xd7, 0x6d, 0xd4, 0x21, 0x05, 0xd4, 0xcf, 0xf3, 0x54, 0xdc, + 0x49, 0x5f, 0x5a, 0x2a, 0x37, 0x19, 0x89, 0x61, 0x1d, 0x95, 0x17, 0x8b, + 0x09, 0x95, 0x5d, 0x9f, 0xde, 0x86, 0x03, 0x93, 0x76, 0xec, 0x54, 0xec, + 0x13, 0xc3, 0xf9, 0x38, 0x8f, 0xa9, 0x11, 0xf0, 0x9a, 0x0e, 0x5e, 0x38, + 0x69, 0xeb, 0x62, 0x41, 0x9e, 0xd0, 0x1b, 0x59, 0x8c, 0xfd, 0x16, 0xfa, + 0xd8, 0x99, 0x0d, 0x83, 0x7e, 0xba, 0x5b, 0xc6, 0x59, 0xe1, 0xae, 0xba, + 0xb9, 0xb8, 0xba, 0xa5, 0x4d, 0x20, 0x00, 0xc9, 0x0c, 0xe1, 0x77, 0xdf, + 0xc4, 0x95, 0xca, 0x7c, 0xa5, 0xef, 0x0a, 0xed, 0x9b, 0x31, 0x06, 0xe1, + 0xc9, 0xa3, 0x88, 0x0a, 0xcc, 0x3d, 0xc8, 0xb6, 0x01, 0xe2, 0xa9, 0x29, + 0x03, 0x8a, 0x28, 0xf8, 0x0d, 0x70, 0x77, 0xb9, 0xe1, 0x1b, 0x06, 0x19, + 0x86, 0xc1, 0xd3, 0xcf, 0x6b, 0x9c, 0x09, 0x70, 0x50, 0xed, 0xb5, 0xf6, + 0x69, 0xcc, 0xac, 0x30, 0x6a, 0x1f, 0x1d, 0xe6, 0x75, 0x33, 0xab, 0x55, + 0x48, 0xfa, 0x81, 0xb8, 0x06, 0x3a, 0x78, 0xee, 0xde, 0xef, 0xe2, 0x17, + 0xc4, 0x3e, 0xe5, 0x22, 0xa7, 0xd1, 0x45, 0x5b, 0x57, 0xb0, 0xde, 0x69, + 0x30, 0xd1, 0x9a, 0xd7, 0x6b, 0x0e, 0x7a, 0x30, 0x0d, 0xb5, 0xec, 0x60, + 0xa7, 0x05, 0x87, 0x42, 0x4b, 0x92, 0x1f, 0x68, 0x8e, 0x1a, 0x90, 0x84, + 0x27, 0x2a, 0xc0, 0xd2, 0xff, 0xbc, 0x8e, 0x34, 0x53, 0x9d, 0x04, 0x50, + 0xcb, 0x79, 0xd9, 0x55, 0xd5, 0x4d, 0x3c, 0xe2, 0xb4, 0x9b, 0x57, 0x07, + 0x1f, 0xce, 0xd0, 0xa7, 0x84, 0xe1, 0xb7, 0x3a, 0xaf, 0xc5, 0x67, 0x64, + 0xbc, 0x02, 0xbe, 0xb0, 0x65, 0x7e, 0xb0, 0x4c, 0xc2, 0x2d, 0xcd, 0xf8, + 0x60, 0xcb, 0xfe, 0xd1, 0x8d, 0x14, 0x5a, 0xd3, 0x38, 0xd4, 0x71, 0x5a, + 0xca, 0xbb, 0xfe, 0x0e, 0x54, 0xf9, 0xb4, 0x25, 0xa5, 0x71, 0x13, 0x95, + 0x14, 0xdc, 0x86, 0xb8, 0x21, 0xa7, 0x2e, 0x13, 0xc6, 0x2f, 0xce, 0xe7, + 0x6c, 0xb8, 0x0d, 0xc9, 0xe4, 0xc4, 0x64, 0x12, 0x78, 0x1c, 0x95, 0x92, + 0xc2, 0xec, 0xaa, 0xd3, 0xc3, 0x3a, 0xd2, 0xe8, 0x95, 0xf0, 0x6b, 0x03, + 0x8c, 0xcf, 0x6b, 0xdb, 0x21, 0xa0, 0xcf, 0xf4, 0x05, 0xc8, 0xe7, 0x77, + 0x05, 0x55, 0x7b, 0x6b, 0xfa, 0x96, 0xf1, 0x7c, 0x30, 0x62, 0x75, 0xbe, + 0x6e, 0xea, 0xba, 0x9f, 0x40, 0x2e, 0x9a, 0x86, 0x93, 0xcc, 0x38, 0xf7, + 0xee, 0xd8, 0xbb, 0x24, 0xcd, 0x85, 0x3e, 0x85, 0x16, 0x8c, 0x33, 0x23, + 0x73, 0xe6, 0x43, 0xc4, 0x67, 0xbf, 0xef, 0x85, 0xb1, 0x44, 0xf9, 0x55, + 0x93, 0x4d, 0x0b, 0x8e, 0xc1, 0x42, 0x13, 0xc6, 0xc8, 0x09, 0x63, 0xab, + 0xb3, 0xc7, 0xc4, 0xa4, 0x8b, 0x72, 0xfb, 0xa5, 0x99, 0xa1, 0x5d, 0x07, + 0x02, 0x82, 0x56, 0x11, 0x3c, 0xc2, 0x5a, 0x55, 0xf9, 0x3a, 0x93, 0x61, + 0x89, 0x46, 0xb7, 0x6a, 0x42, 0x76, 0x1e, 0x70, 0xde, 0xd9, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x32, 0xc1, 0x61, 0xaa, + 0xdb, 0xe9, 0xae, 0x88, 0xcb, 0xf7, 0x28, 0xdd, 0x82, 0x62, 0x61, 0x41, + 0x4e, 0xbb, 0xf9, 0xb7, 0xe8, 0x81, 0x99, 0x18, 0xe2, 0xa7, 0xb4, 0x7c, + 0xb7, 0x08, 0x44, 0x6f, 0x24, 0xb3, 0xda, 0x57, 0x62, 0x29, 0xc7, 0xa6, + 0x84, 0xb1, 0x5d, 0xc5, 0x00, 0x4c, 0x30, 0x16, 0xf0, 0x0a, 0x74, 0x73, + 0xec, 0xaf, 0xb5, 0xde, 0xb0, 0xa7, 0x75, 0x22, 0x8f, 0x9e, 0x43, 0x01, + 0x68, 0xae, 0x91, 0xeb, 0x46, 0x52, 0x3f, 0x2c, 0x4e, 0xc5, 0xd0, 0xc8, + 0x15, 0xea, 0x99, 0xc2, 0x37, 0x5b, 0x68, 0xb5, 0xce, 0x41, 0x92, 0xbf, + 0xd6, 0xdb, 0x85, 0xad, 0x08, 0xd1, 0x11, 0x93, 0xe8, 0xd4, 0x78, 0x43, + 0x3b, 0x7d, 0xcb, 0x42, 0x84, 0xf3, 0x61, 0x88, 0x9e, 0x6a, 0x73, 0xb9, + 0x78, 0x17, 0x9a, 0x9f, 0xfb, 0x97, 0xcb, 0xd6, 0xb5, 0x3f, 0x00, 0x41, + 0xb0, 0x30, 0x2f, 0x6f, 0x89, 0xdd, 0xfa, 0x13, 0xd1, 0x07, 0xbe, 0x2f, + 0xea, 0x91, 0x62, 0xaa, 0xed, 0xcb, 0xfd, 0x07, 0x82, 0xbb, 0x3f, 0xf4, + 0xa6, 0x94, 0x66, 0x71, 0x20, 0x61, 0xac, 0x84, 0x04, 0x70, 0xf2, 0xd3, + 0xdf, 0xac, 0x44, 0xfd, 0x47, 0x26, 0x81, 0x64, 0xb3, 0xa6, 0x90, 0x2b, + 0xd2, 0x2c, 0xd0, 0x77, 0x81, 0x53, 0x45, 0x78, 0x5f, 0x30, 0x77, 0x91, + 0x83, 0x13, 0x33, 0xd1, 0x91, 0xa6, 0x35, 0x21, 0xcb, 0x26, 0x54, 0x0a, + 0xf7, 0x70, 0x5e, 0xdb, 0xd8, 0x92, 0xc7, 0xdf, 0xf9, 0x2a, 0x46, 0x91, + 0x22, 0x3b, 0xe6, 0xe1, 0x91, 0xeb, 0xa6, 0x78, 0x81, 0x57, 0xf3, 0x04, + 0xdf, 0x34, 0x55, 0x74, 0x0a, 0xfe, 0xf2, 0xbd, 0xb3, 0xeb, 0xa3, 0x8e, + 0x71, 0x15, 0xa9, 0x2f, 0x53, 0xe2, 0xa1, 0x45, 0xdf, 0xe8, 0x29, 0x40, + 0xf1, 0x4b, 0x23, 0xdb, 0x8e, 0xee, 0x19, 0xa8, 0xd4, 0x15, 0x90, 0x8c, + 0x04, 0x46, 0x81, 0x49, 0x92, 0xe5, 0xe1, 0xfe, 0x99, 0x06, 0xfc, 0x3e, + 0x43, 0x58, 0x3b, 0x19, 0x7f, 0xd2, 0x13, 0x65, 0xc2, 0x64, 0x27, 0x6d, + 0x93, 0x6a, 0xcf, 0x48, 0x2a, 0x3d, 0xdd, 0x79, 0x9f, 0x05, 0x32, 0xeb, + 0xfd, 0xb4, 0xd2, 0x1d, 0x16, 0x61, 0x3d, 0x17, 0x4c, 0xb8, 0xad, 0x63, + 0x0e, 0x6b, 0x8a, 0x4a, 0x34, 0x4c, 0xb5, 0x3c, 0x0f, 0x05, 0x28, 0x8c, + 0x8b, 0xdf, 0xf4, 0xa0, 0x49, 0xbf, 0x34, 0x6c, 0x6a, 0x5f, 0x40, 0x95, + 0x48, 0x4b, 0x93, 0x1e, 0x61, 0x6d, 0x58, 0xc3, 0x86, 0x98, 0x70, 0x11, + 0x4e, 0x44, 0x65, 0xc1, 0x0d, 0xea, 0x2f, 0xda, 0x38, 0x16, 0xbd, 0xd4, + 0x7b, 0x3e, 0x31, 0xee, 0x42, 0x4c, 0xdc, 0xe9, 0x8b, 0x1f, 0xa9, 0xcf, + 0xab, 0x60, 0xb5, 0xb1, 0xd2, 0xf2, 0x6a, 0xe9, 0xbc, 0xcc, 0xcb, 0x60, + 0x4a, 0xca, 0x70, 0x79, 0x64, 0x9d, 0x07, 0x1e, 0xdb, 0xef, 0x34, 0xaf, + 0x17, 0x93, 0x6b, 0x60, 0x73, 0x2d, 0x8c, 0x08, 0x27, 0x1e, 0x46, 0x9f, + 0xcb, 0x33, 0xdd, 0x76, 0xef, 0x17, 0x58, 0x9a, 0x5f, 0x82, 0x78, 0x0f, + 0xbf, 0xe7, 0x0f, 0x3a, 0x1e, 0xa8, 0x30, 0xbf, 0xff, 0xc7, 0xc7, 0x82, + 0x8b, 0xc3, 0x65, 0x04, 0xfd, 0x45, 0xc9, 0x88, 0x99, 0x8e, 0x44, 0xc5, + 0x23, 0x1e, 0xbf, 0xf1, 0x95, 0x70, 0x35, 0xe6, 0x56, 0x4a, 0x53, 0xb2, + 0xac, 0x0c, 0xfd, 0xf5, 0x61, 0x26, 0x5b, 0x70, 0xd6, 0x4c, 0xfc, 0x0f, + 0xcc, 0x53, 0x6e, 0x25, 0xca, 0x1d, 0x0c, 0x56, 0xf7, 0x9c, 0x95, 0xf6, + 0x3c, 0x08, 0x0c, 0x64, 0xb1, 0x1c, 0x5c, 0xe6, 0x25, 0xa4, 0xa3, 0xb7, + 0xaf, 0x8b, 0xbc, 0xe1, 0x68, 0xdf, 0x10, 0xab, 0xbb, 0xd5, 0x30, 0x64, + 0x42, 0xf6, 0xe6, 0x9a, 0xb5, 0x59, 0x12, 0x76, 0x92, 0xac, 0x29, 0xe9, + 0x45, 0xdb, 0x2e, 0x62, 0x22, 0x58, 0x24, 0x89, 0xc8, 0x6a, 0x2a, 0xa7, + 0x3f, 0x04, 0x53, 0x4e, 0x07, 0x41, 0x4e, 0x5f, 0x95, 0x5f, 0x6e, 0x14, + 0x5b, 0xa7, 0xa7, 0xd3, 0x5a, 0xa2, 0x95, 0x4a, 0xc8, 0xe9, 0x3c, 0x5a, + 0x84, 0x50, 0xbc, 0xe1, 0x9c, 0x7a, 0x16, 0xe5, 0xc7, 0x04, 0x9d, 0x60, + 0x2e, 0x7d, 0xb3, 0x77, 0x5d, 0x86, 0x2e, 0xac, 0x57, 0x2a, 0x31, 0x26, + 0x23, 0x6e, 0xcc, 0x7f, 0xb8, 0x36, 0x29, 0xa9, 0xa8, 0xd9, 0xc6, 0x75, + 0xee, 0x16, 0x23, 0x27, 0x0f, 0xe1, 0xb0, 0x3d, 0x91, 0x3a, 0x26, 0x4a, + 0x60, 0x72, 0x14, 0xf9, 0x3c, 0x66, 0x66, 0xe8, 0x7d, 0x4a, 0x6f, 0x7e, + 0x63, 0x58, 0x6a, 0x28, 0x78, 0x50, 0xef, 0x3b, 0x9d, 0xeb, 0xb6, 0x4b, + 0x5d, 0x55, 0x80, 0x84, 0x97, 0x9b, 0x74, 0x4b, 0x5c, 0x09, 0x1d, 0xe7, + 0x57, 0xfc, 0x40, 0x3f, 0xa9, 0xbd, 0xdf, 0x61, 0x2a, 0x89, 0x62, 0x51, + 0xfc, 0x24, 0xee, 0xee, 0x97, 0x10, 0xca, 0xb6, 0x0e, 0x8e, 0x71, 0x67, + 0x2a, 0x79, 0x4f, 0xc4, 0xe6, 0x3e, 0x27, 0xc2, 0x9b, 0x85, 0xfd, 0xde, + 0xfb, 0x58, 0x75, 0xf3, 0x1c, 0x31, 0xa2, 0x56, 0x3e, 0xdc, 0x24, 0xf4, + 0x4f, 0xcb, 0x5a, 0x1a, 0x77, 0x5c, 0x28, 0xd1, 0x5a, 0x55, 0xa9, 0x8c, + 0xb5, 0xdd, 0x77, 0x93, 0x58, 0xd8, 0x2f, 0x7d, 0x5a, 0x67, 0xa1, 0x95, + 0x0a, 0xd2, 0x6a, 0x93, 0xa6, 0xf0, 0x5f, 0x7f, 0x0a, 0x29, 0xdb, 0x1d, + 0x8c, 0xa7, 0x12, 0x0a, 0xf4, 0xc9, 0xcd, 0x70, 0xd1, 0xbd, 0x48, 0xd4, + 0x9a, 0xbb, 0xbb, 0x24, 0xbf, 0x52, 0x25, 0xb9, 0x75, 0xc2, 0x17, 0x36, + 0x6f, 0x4a, 0xc0, 0x53, 0x6d, 0x38, 0xfb, 0x7a, 0x60, 0xc8, 0x5d, 0x03, + 0xc1, 0x1c, 0x0c, 0x31, 0xf0, 0x59, 0xed, 0x0a, 0x5f, 0x84, 0xf2, 0x89, + 0x6c, 0xb4, 0xd5, 0x24, 0x2d, 0x2a, 0xda, 0xbe, 0x74, 0x1d, 0x22, 0xe2, + 0xc6, 0xf0, 0x9b, 0x98, 0x5a, 0x41, 0x11, 0x4c, 0x51, 0x97, 0x16, 0xa7, + 0xc9, 0xd8, 0x53, 0x12, 0x53, 0xdd, 0x22, 0xa9, 0xf2, 0xae, 0x52, 0x49, + 0x02, 0xf9, 0x5c, 0x78, 0x00, 0xa2, 0x64, 0xff, 0x91, 0x62, 0x20, 0x6a, + 0x87, 0x6a, 0x40, 0x01, 0x85, 0x30, 0xf5, 0xdd, 0xa7, 0x64, 0x0a, 0x85, + 0x8d, 0x37, 0x99, 0xcb, 0x03, 0xc8, 0x29, 0x56, 0x7e, 0x75, 0x4f, 0xa1, + 0xc3, 0x76, 0xce, 0xdb, 0xa3, 0xb4, 0x7e, 0x91, 0x95, 0xbe, 0x53, 0x0e, + 0x20, 0xc9, 0xe7, 0x71, 0x78, 0xad, 0x3d, 0x4c, 0xbb, 0x59, 0xb9, 0x77, + 0xcf, 0x7d, 0x7b, 0xff, 0x15, 0xdb, 0x1d, 0xae, 0x1f, 0xbe, 0x33, 0x88, + 0x01, 0x04, 0x95, 0xe5, 0xe9, 0x6a, 0x1c, 0xbf, 0xc8, 0xc3, 0x33, 0x3b, + 0xd8, 0x2f, 0x75, 0x4a, 0xc3, 0x6f, 0x09, 0x88, 0x26, 0x46, 0x90, 0x89, + 0x53, 0x12, 0x27, 0xc2, 0x7d, 0x23, 0x6b, 0xc4, 0xe3, 0x0a, 0x0f, 0xc2, + 0x86, 0x6d, 0x20, 0x35, 0x82, 0x33, 0xec, 0xdd, 0xa7, 0x6a, 0xc3, 0xa8, + 0x11, 0xdc, 0x02, 0xd9, 0x05, 0x1b, 0x04, 0x75, 0x92, 0x6c, 0x08, 0x9e, + 0x38, 0x72, 0xd9, 0x7d, 0x9b, 0xbc, 0xfd, 0xca, 0xb8, 0x06, 0x0e, 0x24, + 0x89, 0x90, 0xde, 0x52, 0xe4, 0xd1, 0xcc, 0x99, 0x87, 0x0b, 0x87, 0xbb, + 0x5c, 0xa9, 0xab, 0xec, 0xb5, 0xe4, 0xdd, 0x5d, 0xfa, 0xb1, 0x97, 0x5f, + 0x61, 0xf7, 0x58, 0xd6, 0x08, 0x02, 0xf2, 0x51, 0x7c, 0x7a, 0xe6, 0xf1, + 0xcb, 0x43, 0xd0, 0x21, 0x09, 0xb8, 0x82, 0xa9, 0x52, 0xd9, 0xa8, 0x7f, + 0x2b, 0xe1, 0x0f, 0x31, 0xbc, 0x16, 0xa2, 0xce, 0x35, 0x55, 0x2e, 0xd6, + 0xda, 0x38, 0xd9, 0xc2, 0x5e, 0xca, 0x27, 0xd9, 0xa6, 0xd6, 0x4b, 0xa2, + 0x73, 0xc4, 0xce, 0x66, 0x30, 0x60, 0xa2, 0x01, 0xfa, 0xc1, 0xd6, 0xc8, + 0xea, 0xdd, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x70, 0xe2, 0x62, 0x68, 0xff, 0x60, 0x67, 0x64, 0x88, 0xdd, 0x81, 0x79, + 0x82, 0xf5, 0x46, 0xf9, 0x7e, 0x0e, 0xa9, 0x26, 0xf6, 0xcf, 0x5d, 0xef, + 0x10, 0x11, 0xe1, 0x71, 0x72, 0x77, 0xcf, 0x02, 0x7b, 0xf1, 0x6e, 0xc4, + 0xb4, 0xfa, 0x2a, 0x12, 0xfe, 0x7e, 0x3c, 0x66, 0xef, 0x41, 0x98, 0x3a, + 0x1f, 0xa9, 0x14, 0x8f, 0x46, 0x22, 0xa0, 0xc2, 0xee, 0x93, 0x25, 0x34, + 0xf2, 0xb7, 0x6d, 0x0a, 0x36, 0xde, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0xd4, 0x17, 0x62, 0x25, 0xfd, 0x5b, 0x75, 0xeb, + 0xec, 0x06, 0xc9, 0x39, 0x86, 0x6d, 0xc5, 0x60, 0x2d, 0x33, 0x3d, 0xce, + 0x6a, 0x9f, 0x07, 0x3b, 0xb9, 0x70, 0x0f, 0xc7, 0x13, 0x46, 0x35, 0x46, + 0x26, 0xe4, 0xbc, 0x6e, 0x54, 0x89, 0x29, 0xd5, 0xa4, 0x94, 0xa0, 0x3a, + 0x7a, 0x61, 0xcf, 0xd1, 0x48, 0x27, 0x7a, 0x72, 0x95, 0xde, 0x93, 0xd1, + 0x19, 0x1f, 0xc9, 0xc8, 0x8f, 0x0d, 0xce, 0x34, 0x03, 0x39, 0x0a, 0x92, + 0x16, 0x09, 0xc4, 0x49, 0xf9, 0x30, 0x2e, 0x19, 0xd1, 0x69, 0x7e, 0x78, + 0x00, 0x25, 0x30, 0x6f, 0x6b, 0xe1, 0xbe, 0xad, 0xb2, 0x05, 0xde, 0xc7, + 0xc2, 0xf7, 0xd5, 0xa7, 0x4d, 0x03, 0x6f, 0x6b, 0xcd, 0xcb, 0x42, 0xfa, + 0x88, 0x16, 0xd5, 0xa6, 0x60, 0x08, 0xd4, 0xa5, 0x5b, 0x3b, 0x7b, 0xa2, + 0xca, 0xa3, 0xa2, 0x5d, 0x63, 0x7f, 0xc0, 0x37, 0xc5, 0x7e, 0x99, 0x04, + 0x5d, 0x9a, 0xb9, 0xa5, 0xac, 0xd1, 0xe2, 0x5d, 0xb2, 0x2b, 0x7e, 0xbb, + 0xb9, 0x66, 0x13, 0xa7, 0x30, 0xbf, 0x80, 0x0c, 0x2b, 0x8d, 0x45, 0xe1, + 0x8d, 0x96, 0x25, 0x27, 0x47, 0x3d, 0x21, 0x7d, 0x1c, 0x42, 0xac, 0x31, + 0x26, 0x47, 0x59, 0xb3, 0x44, 0x85, 0xf2, 0x8e, 0x7d, 0x01, 0x96, 0x6d, + 0xb2, 0x64, 0xc3, 0xfc, 0xa7, 0x82, 0x06, 0x4a, 0x87, 0x75, 0x9b, 0x99, + 0x47, 0x7e, 0xa6, 0x4d, 0x2c, 0x36, 0xff, 0xac, 0x2b, 0x77, 0x96, 0x52, + 0x14, 0x8d, 0x07, 0x0d, 0x28, 0x9d, 0x84, 0xa2, 0xda, 0xd6, 0x45, 0x3a, + 0xd4, 0xe6, 0xb7, 0x9a, 0xf3, 0x34, 0xe3, 0xda, 0x39, 0xdf, 0x35, 0x9c, + 0xe4, 0x87, 0x55, 0xc8, 0x43, 0xd0, 0x61, 0x46, 0x52, 0x2f, 0x75, 0x63, + 0xbb, 0x98, 0x97, 0xeb, 0xfb, 0x15, 0xaf, 0x8e, 0x96, 0xdc, 0xff, 0x0a, + 0x90, 0xda, 0x09, 0x63, 0x28, 0x7b, 0x92, 0x73, 0x0b, 0xd4, 0x2b, 0x72, + 0x2a, 0x86, 0x32, 0xc3, 0xc1, 0x3e, 0xe4, 0x2c, 0x07, 0x89, 0x53, 0xb7, + 0xfe, 0x78, 0x6c, 0x95, 0xb4, 0x62, 0x4d, 0x4b, 0xfe, 0x6c, 0xfc, 0x5e, + 0x4e, 0xa7, 0x8c, 0x07, 0x4f, 0x85, 0x27, 0xe0, 0x7b, 0xd9, 0x7a, 0xe5, + 0x1d, 0xbc, 0x36, 0xda, 0x8e, 0x21, 0xff, 0xb3, 0x60, 0x2c, 0x5e, 0x23, + 0x0f, 0xde, 0x3f, 0xae, 0xa5, 0x3a, 0x50, 0xa9, 0x99, 0x39, 0x45, 0xaf, + 0xd3, 0x5f, 0x4a, 0x15, 0xad, 0x9c, 0x66, 0x7f, 0x92, 0xe0, 0x02, 0x81, + 0x3e, 0x06, 0x6a, 0x5e, 0xd0, 0x0c, 0x42, 0xe7, 0xcf, 0xe2, 0xeb, 0xa3, + 0xe0, 0xf7, 0x2d, 0x8a, 0x21, 0xdb, 0x64, 0x28, 0x2a, 0xb3, 0x2b, 0xc4, + 0xc9, 0xd5, 0x60, 0xaf, 0xfc, 0x15, 0xa1, 0x44, 0x9c, 0x96, 0x04, 0x42, + 0x1c, 0x55, 0x8c, 0xa5, 0xce, 0x80, 0xce, 0x75, 0x64, 0xa9, 0xf6, 0xa5, + 0x5a, 0x0f, 0x8a, 0x4b, 0x8b, 0x72, 0xcf, 0x3e, 0xd7, 0xeb, 0xe1, 0xd0, + 0xd3, 0x2d, 0x04, 0x6c, 0x9e, 0x02, 0x75, 0x43, 0x5c, 0xc1, 0x57, 0x66, + 0xd9, 0x14, 0x5b, 0x08, 0x10, 0x44, 0x8d, 0x8e, 0x89, 0xd1, 0x65, 0x27, + 0x2a, 0x0b, 0x99, 0x6f, 0x09, 0xa6, 0x20, 0xa5, 0x75, 0x24, 0xe4, 0xf7, + 0xf5, 0xe0, 0xed, 0x79, 0x37, 0x18, 0x13, 0x1c, 0xd9, 0xd1, 0xf5, 0x69, + 0x0c, 0xa5, 0x02, 0xdf, 0x6a, 0xfd, 0x2e, 0x35, 0x8e, 0xd0, 0x41, 0x91, + 0x61, 0x0f, 0x5c, 0xdd, 0x70, 0xbf, 0x1c, 0x49, 0xcb, 0xe9, 0xc9, 0x33, + 0xc4, 0x99, 0x1e, 0x8b, 0x75, 0x48, 0xc2, 0x58, 0xa4, 0x70, 0x1f, 0xbb, + 0xcd, 0xd3, 0x0e, 0x79, 0x25, 0xbe, 0x53, 0xfa, 0x32, 0x32, 0xf6, 0xb9, + 0xf0, 0x0a, 0x52, 0x5b, 0xe0, 0x69, 0xff, 0x43, 0xda, 0x98, 0x1f, 0xee, + 0x54, 0x60, 0xf8, 0x24, 0x43, 0xc5, 0x37, 0x72, 0xd1, 0xfc, 0x99, 0x9a, + 0x3e, 0x24, 0xe0, 0xd9, 0xc2, 0x61, 0x47, 0xb3, 0x26, 0x09, 0x85, 0x74, + 0xa1, 0x2b, 0x4a, 0x70, 0xd0, 0x1b, 0x90, 0x03, 0x25, 0xd9, 0x22, 0xc2, + 0x16, 0x22, 0x3a, 0x62, 0x20, 0xd4, 0x13, 0xce, 0xa2, 0xc7, 0x02, 0xfb, + 0x9a, 0xbf, 0xf1, 0x1c, 0x80, 0x01, 0x97, 0x90, 0x7f, 0x5a, 0x98, 0x70, + 0x30, 0x61, 0x77, 0xe5, 0xd4, 0x3b, 0x03, 0x42, 0x57, 0x31, 0x5e, 0xc6, + 0x64, 0xe1, 0xf4, 0x64, 0x77, 0x21, 0x9b, 0x44, 0x1c, 0xd9, 0x8c, 0x95, + 0x8a, 0xf1, 0xcb, 0x82, 0xac, 0xc1, 0x26, 0x31, 0xf2, 0x22, 0x41, 0xab, + 0xbb, 0x23, 0xd3, 0x8d, 0xcc, 0x5c, 0x9d, 0x9b, 0x1d, 0x9c, 0x4d, 0xf3, + 0x62, 0xde, 0x15, 0x6a, 0x94, 0x8d, 0x24, 0xe7, 0x52, 0x8d, 0x2a, 0xa4, + 0x1d, 0x54, 0x5a, 0xda, 0xaf, 0xab, 0x05, 0x27, 0x4b, 0xbb, 0xb4, 0xda, + 0x0c, 0xb9, 0x20, 0xb3, 0xaf, 0x4a, 0xeb, 0x37, 0xe5, 0x43, 0xe4, 0xc1, + 0xf6, 0x9e, 0xf8, 0x6c, 0xd8, 0xa1, 0x0c, 0xf9, 0xd1, 0x4b, 0x96, 0xa0, + 0x6d, 0x38, 0x64, 0x41, 0xd3, 0x14, 0xfb, 0xad, 0x89, 0xa9, 0xf7, 0x36, + 0x01, 0x0f, 0xbe, 0x8e, 0xd7, 0x76, 0xc6, 0x70, 0x22, 0x32, 0x8b, 0x08, + 0xca, 0x95, 0xbf, 0xcf, 0x5e, 0xb8, 0xc0, 0x3f, 0xd9, 0xaa, 0x84, 0xab, + 0x30, 0x5b, 0xe3, 0x7a, 0x61, 0x32, 0xe5, 0x54, 0x01, 0x5e, 0xb6, 0x1c, + 0x9c, 0x78, 0x52, 0x2a, 0xa7, 0xf5, 0x29, 0xa6, 0x0f, 0x14, 0xa5, 0x3a, + 0x34, 0xd4, 0xf5, 0xc2, 0xb2, 0x8d, 0x12, 0x7b, 0x8a, 0x64, 0x00, 0xfd, + 0x02, 0x0e, 0x02, 0x26, 0x5a, 0xb9, 0xeb, 0xfd, 0x30, 0xce, 0x51, 0xec, + 0x5f, 0xbc, 0xee, 0x53, 0x21, 0xec, 0x0e, 0xee, 0xc4, 0x28, 0x1a, 0xec, + 0x2a, 0x39, 0x4e, 0xe1, 0x50, 0x11, 0x3f, 0x16, 0xdd, 0xbf, 0xaf, 0x3e, + 0xbe, 0xd4, 0xfe, 0x34, 0x1e, 0x62, 0x3f, 0x5a, 0xea, 0x05, 0xfc, 0xd5, + 0x45, 0x08, 0x47, 0xce, 0x38, 0x3f, 0x75, 0x7e, 0x0c, 0x3a, 0x2a, 0x14, + 0xa7, 0x61, 0xba, 0x3a, 0xa1, 0x41, 0xa2, 0x72, 0x19, 0xfa, 0x33, 0x43, + 0xa7, 0xf4, 0x4e, 0x5b, 0xf9, 0xb1, 0x45, 0x16, 0x57, 0x8e, 0xb1, 0xad, + 0x7d, 0x88, 0xd3, 0x93, 0xa2, 0x08, 0xf3, 0x96, 0x4d, 0x84, 0x63, 0x08, + 0xfa, 0x9d, 0xf3, 0x04, 0x33, 0xbd, 0x7e, 0x7a, 0xc7, 0x63, 0xc5, 0x31, + 0x5a, 0x82, 0x33, 0x90, 0x56, 0x44, 0xe9, 0xd3, 0xc4, 0xd4, 0x76, 0x29, + 0x2f, 0xdb, 0xa3, 0x9d, 0xff, 0xd4, 0xd2, 0xb1, 0xce, 0xf1, 0xcb, 0x7f, + 0x10, 0x3b, 0x90, 0xa4, 0x1b, 0xa0, 0x9b, 0xa7, 0xfa, 0x27, 0x40, 0x11, + 0x35, 0xc9, 0x7f, 0x01, 0x97, 0x76, 0x9f, 0x33, 0xc5, 0xd6, 0x8d, 0x20, + 0x07, 0x73, 0x93, 0x0b, 0x24, 0x88, 0x4e, 0x73, 0x68, 0x79, 0x92, 0x20, + 0x2a, 0x71, 0xed, 0x22, 0x0b, 0xfb, 0x42, 0xb5, 0xd9, 0xc3, 0xaa, 0xed, + 0x45, 0x03, 0x64, 0xde, 0x6f, 0x25, 0x8e, 0x3b, 0x9a, 0xef, 0xc5, 0x63, + 0xc2, 0x7f, 0x34, 0xd0, 0x1b, 0x20, 0xa3, 0xab, 0x9d, 0x54, 0x41, 0x0e, + 0x7b, 0x2e, 0x96, 0x12, 0x75, 0x58, 0xdf, 0xd5, 0xaa, 0x3c, 0xf2, 0x26, + 0xc1, 0xf1, 0x18, 0x37, 0x56, 0xf2, 0xd2, 0x86, 0x6f, 0xd4, 0x9f, 0x57, + 0x2b, 0x32, 0xe9, 0x08, 0x94, 0x53, 0x40, 0xc5, 0x4d, 0x77, 0x39, 0xc6, + 0x4c, 0x63, 0x53, 0xf9, 0xbf, 0x35, 0x08, 0xc5, 0x0d, 0xd0, 0x89, 0x82, + 0xa7, 0x2d, 0x6a, 0xb4, 0x22, 0xb1, 0x10, 0x7f, 0xcf, 0x2e, 0x21, 0x27, + 0x9c, 0x12, 0xc6, 0x0e, 0xca, 0xd2, 0x32, 0xb1, 0x6d, 0xfd, 0x59, 0x12, + 0x23, 0x60, 0x46, 0x89, 0xe0, 0x75, 0x5e, 0xc9, 0xf4, 0x3d, 0x8a, 0x89, + 0xd4, 0x23, 0xc2, 0xbe, 0x30, 0x32, 0x4a, 0x95, 0x42, 0xe2, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xa7, 0x0b, 0x48, 0xe2, + 0xeb, 0xd7, 0x12, 0x42, 0x4c, 0x71, 0xfb, 0x25, 0x17, 0x23, 0x0e, 0x01, + 0xa6, 0x21, 0xb9, 0x17, 0x6e, 0xf0, 0x24, 0x66, 0x9e, 0x9d, 0x0f, 0x71, + 0xf8, 0x5b, 0x79, 0xb0, 0x1b, 0x1f, 0xe7, 0xa2, 0xc0, 0x17, 0x16, 0x08, + 0x5e, 0x24, 0x7b, 0xf9, 0x7a, 0x1e, 0x70, 0xe2, 0x05, 0x40, 0x16, 0x56, + 0xe7, 0x79, 0xf2, 0x30, 0xa3, 0xdc, 0xe3, 0x7a, 0x7e, 0x22, 0x88, 0xc0, + 0xf7, 0xc8, 0x5c, 0x93, 0x95, 0x86, 0x02, 0x6c, 0x73, 0x76, 0xef, 0x03, + 0x2d, 0xcb, 0xa5, 0x22, 0xfe, 0x05, 0xbb, 0xe6, 0xfd, 0x19, 0x8c, 0x8b, + 0x67, 0x58, 0x81, 0x81, 0x2d, 0x36, 0xd0, 0xc1, 0x20, 0xb2, 0x87, 0x87, + 0xdb, 0xe4, 0xe5, 0xd1, 0xd1, 0xd5, 0x81, 0x34, 0x4c, 0xd6, 0x09, 0xa2, + 0x5d, 0xcc, 0x99, 0x12, 0xa5, 0x06, 0x0f, 0x06, 0x7e, 0xbb, 0x67, 0x26, + 0x69, 0x15, 0x6e, 0x5f, 0xb1, 0x8e, 0xd6, 0x34, 0xfc, 0x4d, 0xd9, 0x03, + 0xb7, 0x5a, 0xf4, 0xaa, 0x03, 0x00, 0x88, 0x6b, 0x5a, 0xc9, 0xf2, 0xfb, + 0x67, 0x72, 0xbc, 0xf7, 0xb9, 0xdc, 0x97, 0xdf, 0x80, 0x91, 0xfa, 0x30, + 0x18, 0x02, 0x89, 0xc7, 0xc9, 0x62, 0x1d, 0xc0, 0x0b, 0xa6, 0xfe, 0x7e, + 0xb9, 0xa9, 0x1f, 0x11, 0x71, 0xe1, 0xd1, 0xfe, 0x8d, 0x90, 0x2c, 0x09, + 0x82, 0x2e, 0x36, 0x79, 0xa5, 0x75, 0x54, 0xfb, 0xd3, 0x3c, 0xb4, 0x18, + 0x2f, 0x4e, 0x3f, 0x37, 0xc4, 0xf8, 0xc5, 0x59, 0xa3, 0xfd, 0x0c, 0x62, + 0x9e, 0xa8, 0x7a, 0x56, 0xc5, 0x97, 0x89, 0x35, 0xc7, 0xb0, 0x29, 0x87, + 0xbf, 0x6a, 0xdc, 0xb1, 0x2f, 0x01, 0xf4, 0x0d, 0x7c, 0x25, 0x95, 0x39, + 0x81, 0xdd, 0x1a, 0x81, 0x36, 0xc0, 0x6b, 0xbf, 0x6b, 0x4d, 0xea, 0x23, + 0xc0, 0x3e, 0x5c, 0x39, 0xe5, 0x6b, 0x59, 0xa0, 0x50, 0x02, 0x99, 0xdf, + 0x4e, 0xe3, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x17, 0x88, 0xf8, 0xda, 0x3d, 0x57, 0x83, 0x63, 0x76, 0xa0, 0x5c, 0x13, + 0x1a, 0x00, 0x64, 0x30, 0x19, 0xfd, 0x2e, 0x9c, 0x64, 0xb6, 0xda, 0x51, + 0x7b, 0x55, 0xe8, 0xc4, 0x67, 0x1b, 0xda, 0xfc, 0x4c, 0xd0, 0x27, 0x58, + 0x56, 0xa1, 0x52, 0xd2, 0xb8, 0xd8, 0xd5, 0x94, 0x69, 0xcf, 0xd0, 0xd5, + 0x72, 0xeb, 0x2b, 0x05, 0xf3, 0x12, 0xa6, 0xac, 0xa6, 0xf7, 0x90, 0x24, + 0x1f, 0x22, 0x97, 0x5e, 0x8b, 0x7c, 0x2c, 0x30, 0x61, 0x11, 0x9b, 0xdf, + 0x83, 0x2b, 0x10, 0x09, 0x42, 0x77, 0x2b, 0xd9, 0x43, 0xb3, 0x27, 0x69, + 0x75, 0xf2, 0x2e, 0x72, 0xed, 0x50, 0xea, 0xbf, 0x7f, 0x47, 0x39, 0x9c, + 0xf8, 0x1e, 0xce, 0x6f, 0xdd, 0xe8, 0x40, 0xc5, 0x14, 0x01, 0x7e, 0xbb, + 0x0f, 0x43, 0x2d, 0x36, 0x70, 0x54, 0xc6, 0xbe, 0x69, 0x24, 0xd1, 0x65, + 0x49, 0x77, 0xf0, 0xd2, 0x99, 0xb4, 0x50, 0x8d, 0x98, 0xcb, 0xbf, 0x7a, + 0x7c, 0x65, 0xd3, 0x46, 0xcf, 0x90, 0x69, 0x56, 0x15, 0xa2, 0xae, 0x11, + 0x94, 0x60, 0xf9, 0x45, 0x17, 0x54, 0x6b, 0xbd, 0xeb, 0xd8, 0x74, 0x41, + 0x5c, 0xf6, 0x49, 0x0a, 0x14, 0xce, 0x43, 0x1f, 0x67, 0xc3, 0x6c, 0xf4, + 0x01, 0xce, 0x3f, 0x85, 0xed, 0x19, 0xa1, 0xf7, 0x1b, 0xf8, 0x46, 0x45, + 0xb4, 0xe9, 0xa7, 0x1f, 0x2a, 0x65, 0x00, 0x2a, 0xd3, 0x8b, 0x6a, 0x3b, + 0xac, 0x78, 0xab, 0xf4, 0xc8, 0x62, 0x76, 0xc8, 0x24, 0xf8, 0xf8, 0x08, + 0xe0, 0x64, 0x00, 0x64, 0x74, 0x9e, 0x55, 0x2e, 0xf8, 0xc9, 0xc8, 0x58, + 0x0e, 0x1f, 0x27, 0x32, 0xfd, 0x30, 0x24, 0x68, 0xc8, 0xa4, 0x8c, 0x1c, + 0xf3, 0xa7, 0x32, 0xae, 0x84, 0x0a, 0x8a, 0x1e, 0x11, 0xce, 0xb2, 0x02, + 0xf1, 0xb3, 0x5f, 0x7d, 0x5e, 0x54, 0x8c, 0xe0, 0xeb, 0x46, 0x6e, 0x8a, + 0x5f, 0x3f, 0x71, 0x47, 0x2a, 0x8a, 0xe6, 0xf0, 0xb0, 0x04, 0x49, 0x64, + 0xb3, 0x7e, 0x16, 0x09, 0x83, 0x5f, 0x12, 0xe0, 0x85, 0xb7, 0x36, 0xc0, + 0x8a, 0xa5, 0xcd, 0xae, 0xc0, 0xb4, 0xa2, 0x62, 0x9b, 0xfa, 0x64, 0x18, + 0x16, 0x8e, 0xb6, 0x50, 0xf2, 0x9b, 0xc4, 0x7d, 0x0c, 0x4c, 0x8b, 0x58, + 0xcf, 0x9b, 0x87, 0x09, 0xb1, 0x37, 0xbb, 0xaf, 0xa7, 0x72, 0x79, 0x81, + 0x09, 0x55, 0xa1, 0x6a, 0x87, 0xb0, 0x7d, 0xc8, 0xb0, 0xc1, 0xa4, 0xa9, + 0xdf, 0xcf, 0x95, 0x77, 0x36, 0x8e, 0x2b, 0xae, 0xeb, 0x4b, 0xf9, 0x2a, + 0x83, 0x6c, 0x53, 0x3c, 0x89, 0xa6, 0x08, 0xae, 0x00, 0x4e, 0xb8, 0xf6, + 0x34, 0x7c, 0xc6, 0x76, 0x87, 0x1a, 0x02, 0xb0, 0x89, 0xa3, 0x0f, 0x00, + 0xc6, 0x7b, 0xeb, 0xf7, 0x95, 0x40, 0xc5, 0x0d, 0x6f, 0x74, 0xd8, 0x21, + 0x2f, 0x9f, 0x24, 0xac, 0x43, 0xdb, 0x3a, 0x39, 0x6c, 0x34, 0x59, 0x62, + 0x66, 0xbc, 0x28, 0x7f, 0x8c, 0x64, 0x62, 0x8c, 0x28, 0x6c, 0xf5, 0x79, + 0x24, 0xb1, 0x00, 0x9c, 0x58, 0x6b, 0x09, 0xef, 0xb0, 0x73, 0xcd, 0x47, + 0xbb, 0x52, 0xfd, 0x26, 0x6a, 0xff, 0xb9, 0xf1, 0xd5, 0x82, 0x59, 0x01, + 0xfa, 0x87, 0x14, 0x24, 0x10, 0xb0, 0xf7, 0xdf, 0xf9, 0x3f, 0x67, 0x19, + 0xbd, 0xc7, 0x85, 0xb0, 0xad, 0x47, 0xa8, 0x4c, 0x3e, 0xb6, 0x2e, 0x8a, + 0xb3, 0xcc, 0x35, 0xa0, 0x48, 0xc7, 0x90, 0x81, 0xb7, 0x53, 0x1c, 0x38, + 0x63, 0xf2, 0x2f, 0xa0, 0x71, 0x82, 0xe2, 0x56, 0xdb, 0x68, 0xe8, 0x5f, + 0xf8, 0x42, 0xf2, 0xf6, 0xb8, 0x10, 0x6b, 0x54, 0x21, 0xa0, 0xc1, 0xfe, + 0xcb, 0xce, 0x12, 0xa2, 0x49, 0x51, 0x86, 0x53, 0x56, 0xec, 0x33, 0xb3, + 0x72, 0xce, 0xa4, 0x46, 0xe3, 0x37, 0xcb, 0xc0, 0x95, 0xaa, 0xe2, 0xa3, + 0xc5, 0xe9, 0x36, 0x40, 0xfe, 0xf7, 0xe2, 0x5a, 0x6d, 0x58, 0x39, 0xb2, + 0x41, 0x5d, 0xe2, 0x71, 0x72, 0xd0, 0xf0, 0x5c, 0x16, 0x88, 0x95, 0x30, + 0x0a, 0xfb, 0x8d, 0xda, 0x14, 0x80, 0xf4, 0x15, 0xf2, 0xf6, 0xac, 0xf3, + 0xd8, 0x8d, 0x13, 0x24, 0x2c, 0x74, 0x60, 0x6e, 0x8c, 0xa1, 0x59, 0xcf, + 0x74, 0x7c, 0x2d, 0x0b, 0xbb, 0x06, 0x5c, 0x9d, 0xcd, 0xf3, 0x1e, 0x4a, + 0xba, 0x3f, 0x9c, 0x4a, 0xc4, 0xd7, 0xf9, 0xf0, 0xa5, 0x56, 0x7f, 0xb0, + 0xa2, 0x57, 0xd0, 0xc3, 0xaa, 0xa7, 0xd0, 0x49, 0xe2, 0x28, 0x9b, 0xc4, + 0x64, 0x0c, 0xe0, 0x71, 0x9c, 0x05, 0x04, 0x95, 0x00, 0x1f, 0x7b, 0xa9, + 0xb9, 0xb3, 0x2b, 0x8f, 0x0b, 0x45, 0x1e, 0x23, 0xaa, 0x27, 0x89, 0x4a, + 0xb0, 0x7d, 0x03, 0xdf, 0xae, 0xdb, 0xcb, 0xc4, 0xec, 0x3b, 0x02, 0xe2, + 0x85, 0x3a, 0xb7, 0x25, 0xfb, 0xab, 0xca, 0xc1, 0x33, 0x00, 0x5b, 0xd2, + 0xcf, 0xb0, 0x11, 0x1d, 0x51, 0xb5, 0x5b, 0xea, 0x94, 0xf7, 0xa0, 0x98, + 0x33, 0xba, 0x58, 0xfc, 0x12, 0xea, 0xdd, 0x89, 0xbd, 0x63, 0x03, 0xbe, + 0x7e, 0x3b, 0x69, 0xc4, 0x9d, 0x57, 0x0f, 0xd6, 0xbe, 0xea, 0x5b, 0xd0, + 0x97, 0x63, 0x89, 0xb0, 0xa0, 0xc0, 0xd6, 0x39, 0xc1, 0x69, 0x12, 0x6a, + 0xfb, 0xac, 0x74, 0x7f, 0xfb, 0xf4, 0x7f, 0x38, 0x44, 0x4c, 0x8a, 0xa2, + 0x41, 0x15, 0xc0, 0x54, 0xc0, 0xed, 0x14, 0x83, 0xef, 0xbc, 0x9c, 0xc7, + 0xdd, 0x21, 0xd6, 0xf0, 0x9b, 0x7f, 0x09, 0xd5, 0x96, 0xe5, 0xf7, 0xc5, + 0xa9, 0xb3, 0x41, 0xb0, 0x9d, 0xeb, 0x49, 0x68, 0x9d, 0x2b, 0xea, 0x47, + 0x80, 0x3b, 0x54, 0xb8, 0xf4, 0x14, 0x5e, 0xd6, 0x66, 0x89, 0x04, 0xb3, + 0x00, 0xa3, 0xa8, 0x32, 0x62, 0x2e, 0xc3, 0x15, 0xc6, 0x93, 0x7d, 0x40, + 0x32, 0xb1, 0x6b, 0x60, 0xd3, 0x52, 0xdf, 0x09, 0x8c, 0x80, 0x2b, 0x01, + 0xe7, 0x97, 0x8d, 0xbb, 0x14, 0xd6, 0x10, 0x15, 0x64, 0x00, 0x4a, 0x2c, + 0x67, 0xca, 0xd0, 0xa1, 0x37, 0x33, 0x7b, 0xa1, 0x2a, 0x5b, 0x5b, 0x78, + 0xf8, 0x2f, 0xdd, 0x76, 0xab, 0x8a, 0xc3, 0xe3, 0x37, 0x00, 0xd1, 0x29, + 0xb0, 0x96, 0x1d, 0x18, 0xbe, 0x5d, 0x32, 0x7e, 0xb7, 0x11, 0xa9, 0x78, + 0x72, 0xa2, 0x2d, 0x29, 0x1c, 0x32, 0xa4, 0xff, 0xc7, 0xce, 0xfe, 0xaf, + 0xb7, 0x17, 0x43, 0xe5, 0x2f, 0xae, 0x45, 0xd3, 0xaf, 0x10, 0xe3, 0xd0, + 0x58, 0xb6, 0xee, 0xee, 0x7a, 0xb5, 0x06, 0x70, 0x26, 0x7e, 0x2d, 0x5b, + 0xd5, 0xe1, 0x7b, 0x9a, 0x37, 0x02, 0xfc, 0x1d, 0x08, 0x4f, 0x1a, 0xf5, + 0x44, 0x63, 0xde, 0x4b, 0x14, 0x68, 0x54, 0x0b, 0x6a, 0x22, 0x4e, 0x02, + 0x65, 0xcd, 0xf4, 0x04, 0xec, 0xcc, 0x8a, 0x0b, 0xe0, 0x59, 0xf8, 0x65, + 0x25, 0x63, 0xed, 0x0f, 0xa6, 0xc5, 0x3c, 0xcb, 0x5d, 0xc5, 0xd8, 0x9f, + 0x5a, 0xd3, 0x88, 0x3d, 0xd4, 0x2c, 0xb3, 0x04, 0xf6, 0x97, 0xc7, 0xe2, + 0xfd, 0xb6, 0xf4, 0x7d, 0x0d, 0xb9, 0x75, 0x7e, 0x9d, 0x81, 0xdc, 0xdf, + 0x8e, 0x90, 0x40, 0x0c, 0x7b, 0x45, 0xfe, 0x68, 0xfd, 0xff, 0x1c, 0xf1, + 0x16, 0x09, 0x33, 0x74, 0x27, 0x7b, 0x4d, 0xd9, 0x9b, 0x48, 0x6d, 0x84, + 0xeb, 0x96, 0x8f, 0x4b, 0x82, 0x73, 0xd5, 0x69, 0x7d, 0x14, 0x45, 0x8c, + 0xb8, 0x71, 0x87, 0x70, 0x09, 0x26, 0xfc, 0x89, 0x6f, 0x0f, 0xb6, 0xc1, + 0xd6, 0xe1, 0xbf, 0xdb, 0x85, 0x8f, 0x94, 0xad, 0x94, 0x01, 0x01, 0xbb, + 0x3f, 0xc0, 0xb5, 0xff, 0xf5, 0xbb, 0x4f, 0x50, 0x09, 0xca, 0x7d, 0x36, + 0x47, 0x66, 0x9a, 0x8c, 0xee, 0x84, 0x73, 0x9a, 0x1f, 0x49, 0x75, 0xb4, + 0xab, 0x66, 0xf7, 0x3b, 0xfe, 0x81, 0x67, 0xc9, 0xd1, 0x16, 0xde, 0x1f, + 0xc2, 0x24, 0xed, 0x6a, 0x5a, 0xe7, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x18, 0x00, 0x00, 0xc5, 0xd7, 0x14, 0x84, 0xf8, 0xcf, 0x9b, 0xf4, + 0xb7, 0x6f, 0x47, 0x90, 0x47, 0x30, 0x80, 0x4b, 0x9e, 0x32, 0x25, 0xa9, + 0xf1, 0x33, 0xb5, 0xde, 0xa1, 0x68, 0xf4, 0xe2, 0x85, 0x1f, 0x07, 0x2f, + 0xcc, 0x00, 0xfc, 0xaa, 0x7c, 0xa6, 0x20, 0x61, 0x71, 0x7a, 0x48, 0xe5, + 0x2e, 0x29, 0xa3, 0xfa, 0x37, 0x9a, 0x95, 0x3f, 0xaa, 0x68, 0x93, 0xe3, + 0x2e, 0xc5, 0xa2, 0x7b, 0x94, 0x5e, 0x60, 0x5f, 0x10, 0x85, 0xf3, 0x23, + 0x2d, 0x42, 0x4c, 0x13, 0x29, 0xc8, 0x8d, 0x78, 0x6e, 0xd6, 0x8c, 0xe6, + 0xfc, 0xb6, 0x2a, 0xa6, 0x3b, 0xf9, 0xab, 0x61, 0x7c, 0x08, 0x8a, 0x3b, + 0x70, 0xbe, 0x57, 0xaa, 0xda, 0x1f, 0x33, 0x4a, 0x70, 0x17, 0x25, 0x0d, + 0x3f, 0x60, 0x3d, 0xc8, 0x2e, 0xbd, 0x3b, 0x12, 0x0b, 0x63, 0x5e, 0x3f, + 0xf5, 0x6b, 0x1f, 0x0b, 0xd9, 0x33, 0x85, 0x23, 0x71, 0x24, 0x9a, 0xb3, + 0xdf, 0x5c, 0x1f, 0xef, 0x14, 0x33, 0xc8, 0x66, 0x85, 0xb7, 0xf0, 0x56, + 0x68, 0x1d, 0x51, 0x52, 0xaf, 0x80, 0x3c, 0xe2, 0x59, 0x06, 0xf1, 0xd1, + 0x9f, 0xb6, 0xc6, 0x80, 0x4e, 0x06, 0xea, 0x28, 0xab, 0x17, 0x8f, 0x45, + 0x7a, 0xf6, 0xb4, 0x93, 0xb7, 0x43, 0x9e, 0xc6, 0xd4, 0x29, 0x00, 0x62, + 0xab, 0x51, 0x7a, 0x72, 0xe5, 0xc1, 0xd4, 0x10, 0xcd, 0xd6, 0x17, 0x54, + 0xe4, 0x20, 0x84, 0x50, 0xe4, 0xf9, 0x00, 0x13, 0xfd, 0xa6, 0x9f, 0xef, + 0x19, 0xd4, 0x60, 0x2a, 0x42, 0x07, 0xcd, 0xd5, 0xa1, 0x01, 0x6d, 0x07, + 0x01, 0x32, 0x61, 0x3c, 0x65, 0x9a, 0x8f, 0x5d, 0x33, 0xf3, 0xcb, 0x29, + 0x0b, 0x8c, 0xe7, 0x3b, 0x83, 0x44, 0xb1, 0x3a, 0x4f, 0x8e, 0x09, 0x15, + 0x14, 0x69, 0x84, 0xa1, 0xbb, 0x15, 0xfd, 0xea, 0xde, 0xbe, 0x5b, 0x6a, + 0xc0, 0x95, 0x04, 0x46, 0x4d, 0x8a, 0xaa, 0xac, 0xbc, 0x2f, 0xad, 0x12, + 0x15, 0x8a, 0x53, 0x4c, 0x94, 0xb8, 0xca, 0x42, 0x96, 0x3a, 0xf4, 0x7a, + 0x18, 0x9d, 0x5b, 0x24, 0x9a, 0xce, 0xa8, 0x99, 0xd4, 0x37, 0x32, 0xf6, + 0xf2, 0xac, 0xaf, 0x3f, 0xf5, 0x3b, 0xfe, 0xda, 0x13, 0x9a, 0xab, 0x4f, + 0x55, 0xc0, 0x2c, 0x21, 0x2b, 0x65, 0x71, 0x1f, 0xc5, 0x04, 0x32, 0xc9, + 0x94, 0xe5, 0xfa, 0x6f, 0xd8, 0x2a, 0xbc, 0x70, 0x85, 0x55, 0xdc, 0x62, + 0xb7, 0x3a, 0x20, 0x0e, 0xe7, 0x67, 0x3c, 0xfe, 0xcb, 0x83, 0x6a, 0x15, + 0x6e, 0x4a, 0x35, 0x65, 0xea, 0xc1, 0xb9, 0x4d, 0x35, 0xf9, 0x4b, 0xcf, + 0xd8, 0xfd, 0xa5, 0xff, 0xff, 0x67, 0x70, 0x04, 0xae, 0xa2, 0xa4, 0x12, + 0x4b, 0x83, 0x4f, 0xc2, 0x96, 0xf0, 0x21, 0x2b, 0x14, 0x21, 0x73, 0x42, + 0x14, 0x99, 0x07, 0xe5, 0xa9, 0x52, 0x4c, 0xeb, 0xbe, 0xc3, 0x11, 0x2e, + 0x27, 0xda, 0x69, 0x94, 0xd5, 0xf6, 0xc6, 0x77, 0x0a, 0x00, 0x5d, 0x9a, + 0x82, 0xaa, 0x21, 0xfc, 0x86, 0x9b, 0xd0, 0xc4, 0xc4, 0x1f, 0x53, 0x41, + 0x7a, 0x92, 0xab, 0x1c, 0x12, 0xf6, 0xd5, 0x48, 0xfb, 0x29, 0x4d, 0xb4, + 0xd2, 0x12, 0xee, 0xc5, 0xea, 0x18, 0x33, 0xf1, 0x4d, 0x0a, 0x10, 0x43, + 0xa5, 0x35, 0xb1, 0x63, 0xc4, 0xfb, 0x38, 0x1e, 0xef, 0xac, 0x3f, 0x97, + 0x41, 0xc6, 0x96, 0x3e, 0x60, 0x13, 0xc8, 0xe3, 0xbe, 0x61, 0xe9, 0xb6, + 0x26, 0x16, 0x14, 0xf8, 0x82, 0x0d, 0x6e, 0x75, 0x2f, 0xd7, 0x9c, 0x3a, + 0x4a, 0xda, 0xd8, 0x2b, 0x35, 0xd4, 0x20, 0x32, 0xd4, 0x4f, 0x0f, 0xe4, + 0xdc, 0xd5, 0x0f, 0xfe, 0xa6, 0x81, 0x28, 0xb4, 0x24, 0x3e, 0xb7, 0x0f, + 0xb0, 0xb2, 0x5b, 0x05, 0x76, 0xbb, 0x24, 0x49, 0x6a, 0x01, 0x68, 0x3f, + 0x03, 0x96, 0xbc, 0x0c, 0x77, 0x48, 0x5f, 0xe8, 0x39, 0xf4, 0xb0, 0x84, + 0x42, 0x0e, 0x6a, 0xb9, 0xab, 0xf2, 0x95, 0x97, 0xa7, 0x5e, 0x29, 0x34, + 0x9d, 0x50, 0xc0, 0x4b, 0x40, 0x72, 0xa1, 0x7c, 0x79, 0x5e, 0x95, 0xbe, + 0xd6, 0x17, 0x43, 0x0a, 0xc9, 0x27, 0x25, 0x43, 0xd7, 0x99, 0xd5, 0x48, + 0xd8, 0x98, 0xb5, 0x2b, 0x7f, 0xe3, 0xbd, 0x1d, 0xc0, 0xd1, 0x04, 0xd5, + 0xa4, 0xe1, 0x68, 0xbe, 0x96, 0xf1, 0x2e, 0x5e, 0x37, 0x8d, 0x39, 0x4e, + 0xe4, 0xcc, 0x5e, 0xd7, 0xdd, 0x59, 0x7e, 0xe8, 0xae, 0x48, 0xb5, 0xec, + 0x2c, 0xf7, 0x68, 0x96, 0x00, 0xe5, 0xec, 0x03, 0x6f, 0x98, 0x3a, 0x9a, + 0x4f, 0xd9, 0xf1, 0x2f, 0xfe, 0x76, 0xcf, 0x8f, 0x0b, 0x3d, 0x8a, 0x14, + 0x00, 0x83, 0xcb, 0xca, 0xe3, 0x34, 0x81, 0xb5, 0x91, 0x64, 0x2b, 0x12, + 0x24, 0x86, 0x9c, 0xae, 0x3c, 0x7f, 0x53, 0x22, 0xd4, 0x94, 0x90, 0x44, + 0x6b, 0x35, 0xd2, 0xce, 0x8e, 0x95, 0xe2, 0xbe, 0x46, 0x50, 0x3f, 0x3d, + 0xc3, 0xcd, 0xef, 0x47, 0x99, 0xb5, 0xf2, 0xd4, 0x6f, 0xf4, 0xfa, 0xa2, + 0xfc, 0x1e, 0xe3, 0x99, 0x49, 0xfd, 0x1a, 0x6e, 0x0d, 0xb5, 0xf1, 0xc8, + 0x05, 0x22, 0x29, 0xca, 0x03, 0xb8, 0x15, 0x3b, 0x01, 0x8a, 0x95, 0x74, + 0x48, 0x93, 0x61, 0x35, 0xde, 0xeb, 0xa9, 0xc4, 0x56, 0xa9, 0xd7, 0xde, + 0x4b, 0xe5, 0x4b, 0xa1, 0x42, 0x6a, 0x5f, 0xe3, 0xb2, 0xc7, 0xda, 0xfb, + 0xc7, 0x70, 0x64, 0xe0, 0x68, 0x19, 0xc6, 0x11, 0x77, 0x2b, 0x5f, 0xba, + 0x1d, 0x58, 0x77, 0x98, 0x2c, 0x91, 0xb4, 0xd2, 0xea, 0x1b, 0xdc, 0xe8, + 0xfa, 0x82, 0xf3, 0x6e, 0xac, 0x88, 0x15, 0x16, 0x1a, 0x53, 0xb3, 0x01, + 0x94, 0x03, 0x47, 0x20, 0xdb, 0x71, 0xcb, 0x71, 0xe8, 0x62, 0xad, 0x34, + 0x2b, 0xa3, 0xa5, 0xe9, 0xa6, 0x82, 0x0e, 0x16, 0x61, 0xbc, 0x29, 0x6b, + 0xb1, 0x60, 0x67, 0x80, 0x9a, 0x9f, 0xc4, 0x82, 0xf6, 0xb0, 0x7a, 0x16, + 0x9c, 0x25, 0x04, 0xeb, 0xfd, 0xe0, 0x18, 0xd3, 0xfc, 0xeb, 0xe1, 0x3c, + 0x2b, 0x29, 0x7b, 0x32, 0x4e, 0xd3, 0x6d, 0xe1, 0x27, 0xda, 0xc9, 0x14, + 0x5c, 0x7f, 0xfa, 0x70, 0x41, 0x8e, 0xb4, 0xa3, 0xde, 0x36, 0x92, 0x67, + 0x97, 0xe2, 0xec, 0x85, 0x8b, 0x76, 0x08, 0x3c, 0x32, 0x58, 0xd4, 0x7f, + 0x6f, 0x91, 0x03, 0xdb, 0x19, 0x3e, 0xc4, 0x8b, 0x3c, 0xb7, 0x75, 0x90, + 0x71, 0x7a, 0x21, 0x9d, 0xa7, 0x77, 0xbf, 0xf5, 0x92, 0x57, 0x46, 0x07, + 0xa7, 0xbb, 0x0c, 0x42, 0xca, 0x4f, 0x5a, 0x27, 0x45, 0x69, 0xfe, 0x6d, + 0x78, 0x43, 0x77, 0xc4, 0xb4, 0x43, 0xff, 0x37, 0x0d, 0xb7, 0xfa, 0xe9, + 0x9e, 0x06, 0x70, 0x53, 0xfd, 0xf6, 0xa0, 0x28, 0x84, 0x46, 0xcd, 0x61, + 0xa2, 0x95, 0xc4, 0x1e, 0x6a, 0x13, 0xa1, 0x7f, 0xaf, 0xe1, 0x73, 0x85, + 0xb0, 0x53, 0x9c, 0x08, 0xb6, 0x1d, 0x4d, 0xb4, 0x0b, 0xfb, 0x1f, 0x0c, + 0x7b, 0x17, 0x06, 0x73, 0xa7, 0x22, 0x1f, 0xb0, 0xd8, 0x45, 0x6e, 0xe5, + 0xde, 0x48, 0xb7, 0x9f, 0x5a, 0xa8, 0xd1, 0xc3, 0x04, 0xd1, 0x87, 0xec, + 0x15, 0x3e, 0xd1, 0xc7, 0x57, 0x01, 0x46, 0x4b, 0x28, 0xa8, 0x79, 0x5a, + 0x7e, 0x0b, 0x56, 0x56, 0x28, 0xda, 0x35, 0xea, 0x4c, 0x14, 0x81, 0xae, + 0xc0, 0x0d, 0x12, 0xfe, 0x2d, 0xb7, 0x95, 0x4d, 0xea, 0x78, 0xb6, 0x53, + 0xcf, 0xac, 0x8a, 0xfc, 0xc9, 0x07, 0x9f, 0x93, 0xf0, 0x11, 0x86, 0x13, + 0xe9, 0xca, 0x3d, 0xce, 0xb1, 0xfd, 0x1a, 0x0a, 0x8b, 0x11, 0x82, 0x94, + 0x6a, 0xae, 0xc5, 0x80, 0x6a, 0x3b, 0xa8, 0x7c, 0xb4, 0x53, 0x4e, 0xa9, + 0x04, 0x1a, 0x4f, 0xb0, 0xb9, 0x95, 0x96, 0xa5, 0xfd, 0xce, 0xdc, 0x57, + 0x00, 0x48, 0x16, 0xe2, 0x40, 0xae, 0x04, 0xf5, 0x83, 0x60, 0x23, 0xd9, + 0x8e, 0x59, 0x56, 0x20, 0x50, 0x38, 0xc4, 0xde, 0x88, 0x9f, 0x91, 0x06, + 0xdb, 0x8f, 0x84, 0xa2, 0xaf, 0x61, 0xdd, 0x48, 0x03, 0x4f, 0xc4, 0xb8, + 0xed, 0x12, 0xd2, 0x74, 0x08, 0xb9, 0x51, 0x63, 0xb5, 0xfe, 0x09, 0x7f, + 0x7b, 0x8c, 0x5e, 0xd7, 0x27, 0xe5, 0x79, 0xe6, 0x33, 0x60, 0x54, 0xe1, + 0x21, 0xda, 0xca, 0x8b, 0x81, 0xdf, 0xb6, 0xa7, 0x2e, 0x9d, 0x0f, 0xfc, + 0x05, 0x80, 0x67, 0xcb, 0xc5, 0xdf, 0xc7, 0x13, 0xee, 0xb5, 0x40, 0x8e, + 0xa7, 0x0c, 0xcb, 0xf2, 0x45, 0x15, 0x29, 0xb1, 0xb8, 0x02, 0x23, 0x61, + 0x38, 0xf1, 0x16, 0xa1, 0x0c, 0xa1, 0xc9, 0x40, 0x8c, 0xd0, 0x48, 0x4b, + 0xce, 0x9c, 0x1e, 0x53, 0x40, 0x44, 0xf6, 0x17, 0x16, 0xc6, 0x5c, 0xb0, + 0x2a, 0x29, 0x59, 0x87, 0x67, 0x85, 0xa7, 0x81, 0x84, 0xe9, 0x4f, 0xe5, + 0x4e, 0x13, 0x5a, 0x11, 0xa1, 0x24, 0x62, 0xe9, 0x7a, 0xea, 0x51, 0xaa, + 0x45, 0xf3, 0x1d, 0x2a, 0xaf, 0x01, 0x28, 0x35, 0xda, 0xb4, 0xe7, 0xab, + 0xc1, 0xb9, 0x3c, 0x45, 0xa2, 0x0b, 0x5d, 0x40, 0x09, 0xac, 0x62, 0x16, + 0xd3, 0x1f, 0x9f, 0xc7, 0x1a, 0x56, 0xb7, 0x27, 0xd1, 0x1b, 0xe1, 0xb5, + 0x82, 0x9e, 0xe8, 0xd3, 0x5c, 0x0f, 0xe8, 0x87, 0x61, 0xc6, 0x20, 0xb7, + 0x31, 0x3f, 0x0d, 0xb3, 0x0a, 0x5a, 0xce, 0x06, 0xa5, 0xe9, 0xfd, 0xf3, + 0x29, 0x1a, 0xcd, 0x86, 0x0e, 0x31, 0x29, 0xaa, 0xb7, 0x32, 0xf1, 0x10, + 0x4e, 0x92, 0x12, 0x00, 0xc0, 0xac, 0x50, 0x4b, 0x52, 0x59, 0x51, 0x7c, + 0xa8, 0x0c, 0xf7, 0xcb, 0x16, 0x73, 0x7b, 0x90, 0xa8, 0x57, 0x79, 0xb4, + 0x73, 0x53, 0xd7, 0xed, 0xba, 0x46, 0xc5, 0x06, 0x53, 0x02, 0xc7, 0x58, + 0x4c, 0x09, 0x0c, 0xa5, 0x01, 0x13, 0x18, 0x39, 0x4b, 0x4e, 0xc2, 0x0d, + 0xd6, 0xdf, 0xaa, 0x7e, 0x46, 0xba, 0x6e, 0xcc, 0x25, 0x42, 0xd0, 0xb3, + 0x31, 0xdc, 0xdf, 0x7d, 0xf1, 0xc3, 0x73, 0xca, 0x7a, 0xf6, 0xcb, 0x23, + 0x81, 0x8d, 0xbe, 0x0b, 0xf2, 0x79, 0x8d, 0x14, 0xa4, 0xc8, 0x36, 0x18, + 0x49, 0xc8, 0x0d, 0xd7, 0xc9, 0xdd, 0x35, 0xeb, 0xec, 0x52, 0x56, 0xae, + 0xf2, 0xd2, 0x51, 0x91, 0x39, 0xbc, 0xb0, 0x49, 0xb7, 0xf2, 0x1b, 0x64, + 0x83, 0x5a, 0xa6, 0x97, 0xc2, 0x15, 0x95, 0xdc, 0x11, 0xd2, 0x89, 0xc0, + 0x6a, 0xb1, 0x44, 0x43, 0x38, 0xb6, 0x54, 0x0f, 0xdc, 0xcb, 0xed, 0x26, + 0x27, 0xd9, 0x46, 0x56, 0x4e, 0x6a, 0x54, 0x74, 0x0f, 0x45, 0xfc, 0xb6, + 0x93, 0xab, 0x3c, 0xd1, 0x86, 0x51, 0xaf, 0xa9, 0x4a, 0xc0, 0x9c, 0x78, + 0xc1, 0xb1, 0xc7, 0xf1, 0x9c, 0xd1, 0xd0, 0x32, 0x4e, 0x4b, 0x02, 0x36, + 0x68, 0x38, 0x88, 0x56, 0xc0, 0x2b, 0x12, 0x05, 0x3b, 0xb9, 0xf6, 0xa2, + 0x37, 0xe7, 0xbc, 0x81, 0xf9, 0x75, 0x51, 0x27, 0x56, 0x0d, 0x55, 0xd1, + 0x6a, 0xe0, 0xcf, 0x87, 0x0a, 0x44, 0xc6, 0x57, 0xe1, 0x1b, 0xc0, 0x2c, + 0xcf, 0xab, 0x77, 0xe9, 0x14, 0xf5, 0x34, 0x89, 0xfb, 0xc9, 0xf2, 0x87, + 0x5c, 0x75, 0xba, 0x51, 0x9a, 0x49, 0xe9, 0x23, 0x23, 0xf4, 0xc9, 0xd1, + 0x2f, 0x87, 0xf6, 0x75, 0x38, 0x97, 0x48, 0xb8, 0x30, 0x46, 0x1d, 0x46, + 0x65, 0x03, 0x10, 0xcf, 0xfb, 0x36, 0xf2, 0xb1, 0xaf, 0x31, 0x02, 0x7b, + 0x74, 0xfe, 0x9f, 0x8c, 0x73, 0x04, 0xfd, 0xb5, 0xae, 0x2e, 0x27, 0x9c, + 0xd8, 0x73, 0xbc, 0xc3, 0x4a, 0x76, 0x93, 0x66, 0xf6, 0xb7, 0x90, 0xc4, + 0x42, 0x3d, 0xcd, 0xb5, 0xf1, 0x75, 0xbf, 0xb7, 0xdd, 0x8e, 0xb7, 0xcd, + 0x90, 0x35, 0xf5, 0x95, 0x3d, 0xe4, 0x4e, 0xb0, 0x7c, 0x5f, 0xad, 0xff, + 0x75, 0x38, 0xc4, 0xc7, 0xed, 0xec, 0x70, 0xcc, 0x9f, 0xf9, 0x77, 0xa1, + 0x00, 0x2f, 0xf1, 0xa2, 0xc9, 0x74, 0xdc, 0x18, 0x14, 0xd0, 0x2f, 0x86, + 0x66, 0xa7, 0x5b, 0x39, 0x5c, 0xba, 0x0e, 0x77, 0x16, 0x04, 0xc3, 0x02, + 0x42, 0x3b, 0x66, 0x29, 0xee, 0x65, 0x00, 0xd4, 0x22, 0x5a, 0x77, 0x74, + 0xd4, 0xc3, 0xf3, 0x00, 0xdf, 0x6b, 0xc3, 0x15, 0x89, 0x0e, 0xb1, 0xbc, + 0xac, 0xe8, 0x44, 0x2f, 0x80, 0x34, 0x34, 0x8b, 0x0c, 0x48, 0x45, 0xc2, + 0x6a, 0xa3, 0x67, 0xd7, 0x3d, 0x36, 0xf3, 0x3f, 0xe5, 0xf0, 0x5b, 0xe8, + 0xad, 0x41, 0xd5, 0x82, 0xc1, 0x28, 0xab, 0x77, 0xe8, 0x7f, 0xb3, 0xf6, + 0xd2, 0x0c, 0xe4, 0x03, 0xcf, 0xe4, 0x72, 0xdb, 0x7b, 0x81, 0xf4, 0xf3, + 0x48, 0x74, 0xe1, 0x91, 0xb8, 0xf8, 0x4c, 0x2c, 0x60, 0x99, 0x3e, 0x1e, + 0x4f, 0xaf, 0x12, 0xab, 0x52, 0xef, 0xc7, 0x60, 0xd2, 0xfe, 0x62, 0x55, + 0xc8, 0x18, 0xad, 0x60, 0xa7, 0x5d, 0xde, 0x4d, 0xfc, 0x6d, 0xe1, 0x10, + 0x7c, 0xf9, 0xa2, 0x64, 0x00, 0x16, 0x1f, 0x44, 0x7c, 0xe2, 0x72, 0x37, + 0xd9, 0x92, 0xad, 0xfc, 0x62, 0x53, 0xbe, 0xb6, 0xe0, 0xc8, 0xe0, 0xa2, + 0xef, 0x22, 0x4b, 0x70, 0x3a, 0x4f, 0xc9, 0xed, 0x6b, 0xbc, 0x17, 0x0a, + 0xcf, 0x6a, 0x2c, 0xd3, 0xd2, 0x6b, 0x02, 0x45, 0xfa, 0x9e, 0xc2, 0x21, + 0x28, 0xfc, 0x07, 0x68, 0xd6, 0xb8, 0x9f, 0x2a, 0x0b, 0x7a, 0x0e, 0xbc, + 0x4e, 0xee, 0x84, 0x38, 0xe4, 0x8e, 0x70, 0xc3, 0xc4, 0xad, 0x74, 0x87, + 0x2d, 0x16, 0x4f, 0xa1, 0xf8, 0x20, 0xf5, 0xde, 0xa3, 0xc5, 0x0c, 0x3b, + 0xde, 0x44, 0x48, 0x0f, 0x3c, 0xdc, 0x7e, 0x10, 0x8b, 0x87, 0xc4, 0x3b, + 0xb0, 0x95, 0xbf, 0x61, 0x1e, 0xad, 0x07, 0x52, 0xfd, 0x0b, 0x84, 0xa9, + 0x46, 0xb0, 0x32, 0xd5, 0x22, 0x80, 0x35, 0x26, 0x41, 0xf8, 0x11, 0x72, + 0xb1, 0x31, 0x6f, 0x5a, 0x75, 0xcc, 0x67, 0xe0, 0xb2, 0x50, 0x89, 0xb2, + 0x66, 0x6e, 0xee, 0xa0, 0x41, 0x8d, 0x00, 0x2a, 0xa7, 0x9d, 0xa5, 0x11, + 0x2b, 0x07, 0x95, 0x3a, 0x55, 0x8c, 0x67, 0xb1, 0xe5, 0x2d, 0xd4, 0xd1, + 0x3e, 0x29, 0xed, 0xa5, 0x59, 0x97, 0x7b, 0xdf, 0x92, 0x10, 0x0b, 0x04, + 0x89, 0x27, 0xa0, 0xa2, 0x93, 0x18, 0x7f, 0x47, 0x84, 0x1c, 0xc6, 0xd6, + 0x8f, 0x73, 0x81, 0xa0, 0xfa, 0xe5, 0x3e, 0xd8, 0xbf, 0x56, 0x1a, 0x76, + 0xf4, 0xc4, 0x0f, 0x7a, 0x29, 0x9d, 0x32, 0x5d, 0x41, 0xe0, 0x07, 0xb9, + 0xd3, 0x3f, 0x7e, 0xff, 0x90, 0x89, 0xce, 0xdc, 0xf1, 0x1d, 0x54, 0xb6, + 0x67, 0x7f, 0x4d, 0x71, 0x9a, 0x4a, 0x5f, 0x80, 0x0d, 0x5c, 0x77, 0xd5, + 0x50, 0x7c, 0x41, 0x56, 0x7e, 0x99, 0x0a, 0xeb, 0x66, 0x1f, 0xd2, 0x55, + 0xc3, 0xc6, 0x6c, 0xc5, 0xfc, 0x34, 0x40, 0x2c, 0x05, 0x29, 0x05, 0x7c, + 0xca, 0xe6, 0x8d, 0xd3, 0xb0, 0xca, 0x84, 0x27, 0x50, 0x7c, 0x6b, 0x17, + 0x1b, 0x22, 0xe4, 0x7f, 0xe6, 0x44, 0x94, 0x06, 0x4b, 0xb3, 0xb7, 0xbb, + 0x98, 0x81, 0x44, 0x0b, 0xf5, 0x66, 0xcb, 0xad, 0xf2, 0x9a, 0xe1, 0x47, + 0xf3, 0x97, 0xa9, 0xb2, 0xc2, 0xca, 0xcd, 0x98, 0x78, 0x60, 0xdc, 0x6e, + 0x87, 0x55, 0x47, 0xf3, 0xae, 0x84, 0xdd, 0x9a, 0xe9, 0x1a, 0x63, 0x83, + 0xea, 0x23, 0x09, 0x67, 0x34, 0x83, 0x00, 0x6e, 0x5e, 0x58, 0xb8, 0x89, + 0x04, 0x08, 0x0a, 0x55, 0x9e, 0x78, 0xc9, 0xff, 0xb9, 0xb5, 0x2c, 0xdd, + 0x3b, 0x0c, 0x58, 0x07, 0x8b, 0xb4, 0x6a, 0xc4, 0x64, 0xa3, 0x5e, 0x5b, + 0xfe, 0x4d, 0xd0, 0x74, 0x01, 0x1b, 0xdf, 0x10, 0x45, 0x2b, 0xd6, 0x9e, + 0xa9, 0x60, 0x1f, 0xad, 0x46, 0xa1, 0x8c, 0xf8, 0xf6, 0xa9, 0x8a, 0x27, + 0xea, 0x51, 0x37, 0x84, 0xcf, 0xe5, 0xd7, 0x51, 0xd6, 0x40, 0x39, 0x39, + 0x5f, 0xf6, 0x96, 0x33, 0xd9, 0x86, 0x8d, 0x38, 0xb6, 0x26, 0x04, 0x14, + 0x07, 0x46, 0x3e, 0xd0, 0xc5, 0xf6, 0x0d, 0xa0, 0x47, 0x2b, 0xc8, 0x73, + 0x18, 0x6b, 0xd3, 0x0e, 0x18, 0xcc, 0x43, 0x98, 0xd0, 0xcf, 0x1c, 0xe4, + 0x4a, 0x41, 0x6a, 0x56, 0x2d, 0xf0, 0x93, 0x89, 0x81, 0x6c, 0xce, 0x04, + 0x1a, 0x23, 0x05, 0x91, 0x4f, 0x48, 0x44, 0x3a, 0xaa, 0x03, 0xa5, 0x4a, + 0xa9, 0x20, 0x2c, 0xbe, 0x6a, 0x81, 0xe6, 0xa9, 0xf8, 0xf0, 0x2b, 0x29, + 0xa1, 0xe0, 0xc4, 0xce, 0xf5, 0xda, 0x25, 0x70, 0x49, 0xcc, 0xa0, 0x4b, + 0x24, 0x49, 0x4f, 0x11, 0xc4, 0x3b, 0x22, 0x89, 0x9a, 0xb4, 0xf4, 0xcd, + 0xa3, 0xee, 0xb0, 0x76, 0x13, 0xc4, 0xbb, 0xaf, 0x03, 0x7f, 0x27, 0xf3, + 0x38, 0xbc, 0xde, 0x7c, 0x0c, 0x39, 0x14, 0xb7, 0x14, 0xbb, 0x5c, 0xae, + 0x89, 0xf8, 0xf7, 0xd6, 0x00, 0x78, 0xf4, 0xb0, 0x52, 0x16, 0xf5, 0x54, + 0xc5, 0x93, 0xf7, 0x6d, 0x0d, 0xe8, 0x58, 0xe2, 0xa1, 0xa7, 0xdc, 0x49, + 0xdb, 0xc8, 0x79, 0xbc, 0xc3, 0x97, 0x7b, 0x6c, 0x82, 0x7b, 0xbe, 0xe9, + 0x79, 0xac, 0x4a, 0xa4, 0x7c, 0x49, 0x83, 0x58, 0x3a, 0xe4, 0xf5, 0x68, + 0x5c, 0xb7, 0x7f, 0x2d, 0xfe, 0x6b, 0x96, 0xc7, 0x8b, 0x67, 0xb5, 0xd0, + 0xa1, 0x0a, 0x16, 0x62, 0x64, 0x53, 0xea, 0x29, 0x80, 0x93, 0xf9, 0xd6, + 0xa0, 0xc5, 0x1b, 0x3a, 0x1e, 0xab, 0x51, 0x88, 0xe0, 0x9e, 0xd4, 0xf6, + 0xbf, 0x70, 0x2d, 0x29, 0x2e, 0x08, 0xa9, 0x31, 0x78, 0x0a, 0x15, 0x30, + 0x9f, 0x2e, 0xc8, 0x41, 0x65, 0x8e, 0x97, 0x51, 0x5e, 0x73, 0x46, 0x42, + 0x74, 0x84, 0xfd, 0x9b, 0x4a, 0x8a, 0x68, 0x28, 0x45, 0xd0, 0x5d, 0x65, + 0x08, 0xb3, 0xf5, 0x40, 0x8a, 0x29, 0x8e, 0x70, 0x02, 0x49, 0x6a, 0x01, + 0xd6, 0x41, 0x4a, 0xf8, 0x15, 0xa3, 0x70, 0x59, 0xe9, 0xa2, 0xe2, 0x76, + 0x8c, 0x60, 0x33, 0xb3, 0xfa, 0x8b, 0xb4, 0x90, 0x6f, 0x92, 0xc8, 0x21, + 0x59, 0xc0, 0x3a, 0x30, 0x46, 0xeb, 0x49, 0xd8, 0x85, 0x63, 0x5a, 0x23, + 0x87, 0xe1, 0xa7, 0xc0, 0x1a, 0xb0, 0xc7, 0xc4, 0x40, 0x4d, 0x11, 0x9c, + 0xe3, 0xd4, 0x6b, 0xef, 0x68, 0xc8, 0x2c, 0x31, 0xcd, 0x3e, 0xee, 0x55, + 0x10, 0x67, 0x77, 0x7b, 0x30, 0xc1, 0xd0, 0x23, 0x6c, 0x65, 0x6f, 0xfb, + 0x2e, 0x62, 0x33, 0x42, 0x63, 0xdc, 0xca, 0x86, 0xf1, 0x0e, 0xb3, 0xb0, + 0x69, 0x11, 0x65, 0xe1, 0x6e, 0x6c, 0x03, 0x49, 0x79, 0xe8, 0xf1, 0x2e, + 0x8d, 0x94, 0xc8, 0xa8, 0x98, 0x2d, 0x3f, 0xfe, 0xbd, 0x2d, 0x75, 0x45, + 0xd1, 0x7a, 0x09, 0xf8, 0x90, 0x49, 0xbd, 0x4a, 0x3b, 0xa4, 0xa3, 0x26, + 0xb8, 0x62, 0x66, 0x97, 0xd9, 0xc1, 0xca, 0x12, 0x49, 0xe1, 0x27, 0x93, + 0x4f, 0x60, 0xfa, 0xb3, 0x4f, 0x4c, 0xdb, 0x87, 0x6c, 0x3b, 0x50, 0x47, + 0xe2, 0xd8, 0x5b, 0x13, 0x99, 0xf0, 0x2b, 0xbb, 0x32, 0x33, 0xfd, 0x7d, + 0x15, 0x0f, 0x2c, 0xee, 0x85, 0x83, 0xc0, 0x53, 0x79, 0x3e, 0x51, 0xfe, + 0x7c, 0x06, 0x73, 0x49, 0x49, 0x4f, 0x5a, 0x22, 0x36, 0x8f, 0x30, 0x8a, + 0xef, 0x84, 0xd6, 0x15, 0x26, 0x48, 0xe7, 0x1e, 0xb1, 0xaa, 0x82, 0xd0, + 0xc7, 0x0b, 0x97, 0x7b, 0x6c, 0x2d, 0x49, 0x7e, 0x6d, 0xe7, 0xa3, 0x05, + 0x80, 0xd7, 0x42, 0xa9, 0xc6, 0x66, 0x98, 0x30, 0xe3, 0x8a, 0x79, 0x86, + 0x9c, 0x2b, 0xbc, 0x4a, 0xe6, 0x0d, 0xc5, 0xe5, 0x1a, 0x92, 0xd9, 0xef, + 0x63, 0x52, 0x03, 0x88, 0x36, 0xc5, 0x83, 0x65, 0xf8, 0xf1, 0x87, 0xce, + 0x43, 0xfe, 0x89, 0x58, 0x07, 0x6a, 0xad, 0x85, 0x37, 0x0f, 0xdf, 0x9e, + 0xa5, 0x62, 0xa9, 0xd2, 0x41, 0x3f, 0x7f, 0xb7, 0xf1, 0xe2, 0x58, 0xb5, + 0xda, 0xdf, 0xd1, 0xba, 0x36, 0x2c, 0xe7, 0x43, 0x31, 0x07, 0xc5, 0xf5, + 0x79, 0xc9, 0x31, 0xd7, 0x1d, 0x97, 0x57, 0x9a, 0x8e, 0x3f, 0xac, 0x00, + 0x49, 0x00, 0x2f, 0xad, 0xac, 0xe7, 0x65, 0x7c, 0xbf, 0xec, 0x85, 0x57, + 0xe6, 0xcc, 0x07, 0x34, 0x02, 0x36, 0xa8, 0x6a, 0x9f, 0x3a, 0x9a, 0x2f, + 0x34, 0x93, 0x1f, 0x7d, 0x38, 0x54, 0xe3, 0x54, 0x54, 0xee, 0x84, 0x55, + 0xe1, 0x0d, 0xc1, 0x08, 0x3e, 0x33, 0x9e, 0x2a, 0xc3, 0x6a, 0x83, 0xc4, + 0x75, 0xed, 0xbc, 0x5f, 0xd9, 0x04, 0xd7, 0x77, 0x91, 0xb1, 0xa0, 0xf2, + 0xef, 0x81, 0xb0, 0x8b, 0x53, 0x5f, 0x71, 0xec, 0xa5, 0x0b, 0xbe, 0xf2, + 0x92, 0x7e, 0x0a, 0x34, 0xeb, 0x5d, 0x65, 0xc7, 0xa9, 0x44, 0x10, 0xfb, + 0xd3, 0xef, 0xe1, 0xbc, 0x06, 0x65, 0x68, 0x22, 0xfb, 0x43, 0x2c, 0xcf, + 0x8e, 0x6a, 0x28, 0xdb, 0x0b, 0xf4, 0xaf, 0x01, 0x65, 0x97, 0xd6, 0xe5, + 0x91, 0x20, 0x13, 0x2c, 0xb1, 0xc2, 0xd3, 0xc3, 0x76, 0x90, 0xf8, 0xcd, + 0x00, 0xde, 0x93, 0xf8, 0x4e, 0xcc, 0xdc, 0xca, 0x9a, 0xf0, 0xbd, 0x9b, + 0xd6, 0x57, 0xb1, 0x13, 0xd9, 0xe0, 0xe1, 0x9e, 0x21, 0x74, 0xa9, 0x76, + 0xc0, 0x0c, 0xad, 0x4f, 0x5d, 0xfe, 0x23, 0x32, 0x5a, 0x10, 0x75, 0x5b, + 0x05, 0xdf, 0xdc, 0x5b, 0x94, 0xcb, 0xe1, 0x9f, 0x13, 0x51, 0xf5, 0x50, + 0x36, 0x3b, 0xf2, 0x90, 0x9c, 0x9a, 0xc8, 0x10, 0x88, 0xa9, 0xec, 0x22, + 0x1e, 0x96, 0x70, 0xe8, 0x9e, 0x69, 0xc1, 0x22, 0xd9, 0x14, 0x15, 0x2e, + 0xbc, 0x03, 0x96, 0x9e, 0x1d, 0x00, 0x10, 0x16, 0x4f, 0x56, 0xf0, 0x29, + 0x47, 0x0a, 0x45, 0x34, 0x27, 0x21, 0x3b, 0x67, 0x33, 0xf9, 0xdd, 0x29, + 0x3a, 0xf2, 0xe4, 0x56, 0x34, 0x46, 0xbe, 0xd8, 0x42, 0x29, 0x11, 0x7f, + 0x30, 0xc1, 0xbe, 0xa5, 0xc8, 0x9d, 0x7b, 0x2e, 0x4e, 0xcf, 0xba, 0x91, + 0xb4, 0xbf, 0x0a, 0x04, 0x00, 0x49, 0x83, 0x6b, 0x46, 0x5f, 0x3b, 0xfa, + 0xf7, 0x40, 0x8d, 0x85, 0x47, 0x14, 0x58, 0xb3, 0xa5, 0x66, 0x30, 0xfd, + 0x4a, 0x80, 0xa4, 0x61, 0x3b, 0x7c, 0xb4, 0xcc, 0x34, 0x8c, 0xc6, 0xb6, + 0x10, 0xa9, 0x76, 0xc9, 0x11, 0xd7, 0x8a, 0x51, 0x86, 0x17, 0x89, 0x28, + 0xab, 0xd5, 0x03, 0x88, 0x74, 0x5b, 0x81, 0xbd, 0x3a, 0x57, 0xfe, 0x66, + 0x25, 0xd0, 0x92, 0x15, 0x84, 0x02, 0x0f, 0x51, 0xa8, 0x58, 0xcf, 0x77, + 0x65, 0x10, 0x61, 0xe8, 0xe6, 0xab, 0xb1, 0xba, 0x3b, 0x08, 0xd6, 0xba, + 0x5f, 0xf5, 0x74, 0xc5, 0x07, 0x60, 0xfd, 0xd3, 0xc8, 0x52, 0x4e, 0xdb, + 0xc3, 0xe3, 0x6d, 0x81, 0x20, 0x51, 0x01, 0x9a, 0x5e, 0x32, 0x4e, 0x80, + 0x5a, 0xcb, 0x83, 0xd7, 0xa4, 0xd9, 0xfb, 0xed, 0x3d, 0x80, 0xa1, 0x83, + 0x81, 0x91, 0xc0, 0x0b, 0xff, 0x67, 0xd8, 0x8b, 0xd0, 0x12, 0x0b, 0xd4, + 0x2b, 0x8e, 0x0d, 0x0f, 0xfc, 0xc7, 0xb3, 0xf1, 0xe3, 0xf3, 0x5e, 0x0c, + 0xb6, 0x6b, 0x9d, 0xdc, 0x22, 0x70, 0x31, 0x54, 0xe8, 0x41, 0xfe, 0xa1, + 0xe1, 0x4f, 0xfa, 0x81, 0xfb, 0xae, 0x72, 0x16, 0xb8, 0x87, 0xc9, 0x31, + 0x9d, 0x42, 0x47, 0x4a, 0x20, 0xae, 0x63, 0x16, 0x0d, 0xfa, 0xf1, 0x27, + 0x19, 0x47, 0xee, 0x45, 0x84, 0x29, 0x9a, 0xb6, 0x42, 0xef, 0xbd, 0x15, + 0xa8, 0x34, 0x33, 0x38, 0x9c, 0x9d, 0xbb, 0x5c, 0x03, 0xf3, 0xcf, 0xcf, + 0x6d, 0x2e, 0xd5, 0x88, 0xf8, 0xdd, 0xfc, 0xc0, 0x4a, 0xdb, 0x69, 0xd9, + 0x62, 0x89, 0x24, 0x46, 0xee, 0xa4, 0xb9, 0x95, 0xe6, 0xaf, 0x7d, 0x53, + 0xec, 0x41, 0xae, 0x70, 0xfe, 0x4f, 0x31, 0xe3, 0xa2, 0x59, 0x2c, 0xa1, + 0x53, 0x8b, 0xb6, 0x3b, 0x39, 0xc1, 0xa4, 0xa7, 0x9e, 0xaa, 0x00, 0x60, + 0x9a, 0x5f, 0x56, 0x51, 0xf3, 0x7b, 0x28, 0x84, 0x36, 0x1a, 0xc1, 0x2d, + 0xc8, 0xed, 0xf8, 0x48, 0x48, 0x1d, 0x39, 0x4d, 0x3d, 0xce, 0x30, 0x90, + 0x29, 0x33, 0x6f, 0x9a, 0xce, 0x58, 0xe7, 0x88, 0xac, 0x59, 0xce, 0x85, + 0x5a, 0x52, 0x2b, 0x6c, 0xb7, 0xe9, 0x2e, 0xa9, 0xd9, 0x9a, 0xea, 0x1c, + 0x47, 0xb2, 0x59, 0xff, 0x73, 0x76, 0x21, 0x40, 0xe1, 0xde, 0x32, 0xb8, + 0x73, 0x3d, 0xa5, 0x44, 0x66, 0x79, 0xa1, 0xfe, 0xaf, 0xf6, 0x8a, 0x97, + 0x09, 0x5c, 0x8b, 0x64, 0x38, 0x9f, 0xe1, 0x59, 0x38, 0x18, 0xe9, 0xc0, + 0xd6, 0xa2, 0xac, 0x74, 0xa9, 0xfd, 0x4a, 0x0d, 0xf6, 0x47, 0x00, 0x2b, + 0x09, 0x46, 0x38, 0x1c, 0xa4, 0x9f, 0x63, 0x20, 0x18, 0x75, 0x5a, 0xb8, + 0xc4, 0xbc, 0xd6, 0x6b, 0xc8, 0x14, 0x72, 0x03, 0xe4, 0x05, 0xd4, 0x4e, + 0x66, 0x20, 0x42, 0xa2, 0x8f, 0x96, 0xe7, 0xaf, 0xd3, 0xfb, 0xa8, 0x88, + 0x9b, 0xe3, 0xaa, 0xcd, 0xab, 0xce, 0x8f, 0x07, 0x6d, 0xef, 0x98, 0xce, + 0xdb, 0x42, 0x5b, 0xf4, 0x61, 0x57, 0x62, 0x27, 0x8a, 0x53, 0x5e, 0xf8, + 0x3e, 0xf6, 0x7f, 0xde, 0x5e, 0x3b, 0x1b, 0x13, 0x2e, 0x30, 0x46, 0x4b, + 0x6b, 0xb7, 0xbb, 0x33, 0x31, 0xc0, 0xfa, 0x40, 0xab, 0x68, 0x72, 0xe3, + 0x92, 0x30, 0x47, 0xd6, 0x30, 0x60, 0x42, 0x5b, 0x88, 0x8d, 0xa6, 0x56, + 0xe4, 0xac, 0x33, 0x2e, 0xca, 0x05, 0x1f, 0x60, 0xaf, 0xde, 0x7f, 0xa9, + 0xda, 0x3f, 0xa8, 0x21, 0xf6, 0xfc, 0x98, 0x7d, 0xc4, 0x1e, 0xb0, 0xa9, + 0x56, 0x2d, 0x8d, 0xea, 0x03, 0x51, 0x48, 0xac, 0xe8, 0x22, 0xc7, 0x8b, + 0xef, 0x91, 0x0e, 0xcf, 0x0c, 0xe9, 0x38, 0x43, 0x99, 0xa8, 0x98, 0x4f, + 0xfa, 0xe3, 0x03, 0xa6, 0x4f, 0xd4, 0x0d, 0x98, 0x5b, 0x50, 0x28, 0xd7, + 0xe7, 0x46, 0xd7, 0xad, 0x43, 0xb8, 0x56, 0x2a, 0x2f, 0x7c, 0x39, 0x67, + 0xf4, 0x62, 0x0e, 0xc0, 0xa8, 0x87, 0xb5, 0x81, 0xe2, 0x13, 0x9f, 0xe4, + 0xdd, 0x72, 0xf2, 0x07, 0xca, 0xac, 0x6d, 0xb2, 0x96, 0x53, 0x5a, 0x8f, + 0x66, 0x3c, 0xb4, 0xc1, 0x4f, 0x9a, 0x82, 0x55, 0xcf, 0x0e, 0x27, 0x5f, + 0xc7, 0xd2, 0x28, 0x27, 0x7f, 0x22, 0x6e, 0xa5, 0xe7, 0x32, 0x56, 0x51, + 0x18, 0xe0, 0x85, 0x6d, 0x1f, 0xfc, 0x25, 0x08, 0x18, 0x60, 0x57, 0xfc, + 0x66, 0x94, 0x2c, 0x4c, 0xbe, 0x00, 0xab, 0x9e, 0x73, 0x9b, 0x06, 0xd3, + 0xb5, 0x24, 0xa8, 0x8f, 0xb1, 0x33, 0x99, 0x4c, 0xb4, 0x13, 0x07, 0xcd, + 0x04, 0xdd, 0x77, 0xdc, 0xee, 0x96, 0x02, 0x59, 0xe8, 0x22, 0x07, 0x16, + 0x2e, 0x41, 0xc9, 0xc4, 0x59, 0x70, 0x37, 0x0f, 0x14, 0xc9, 0xcf, 0x90, + 0x57, 0xc2, 0x0d, 0xa3, 0xd7, 0x66, 0xb6, 0x7d, 0x10, 0xd4, 0xfc, 0x18, + 0x66, 0xad, 0xea, 0x5e, 0x64, 0x6c, 0x12, 0x66, 0x3d, 0x96, 0xa5, 0xa8, + 0x9c, 0x49, 0x5c, 0xd4, 0x8d, 0x1c, 0xc3, 0x38, 0xfe, 0x53, 0xc2, 0x71, + 0xd1, 0xc6, 0x41, 0xe2, 0xb9, 0x17, 0x74, 0x6e, 0xcc, 0xf8, 0x72, 0x28, + 0x38, 0x4e, 0x54, 0x9b, 0x0e, 0xa3, 0x3a, 0x43, 0x5c, 0xd5, 0x83, 0x06, + 0xbb, 0x46, 0x16, 0x6e, 0xe3, 0x8a, 0xd5, 0x1e, 0x7f, 0x88, 0x62, 0xac, + 0x35, 0x89, 0xfb, 0xbe, 0x96, 0x1d, 0x87, 0x37, 0xb7, 0x91, 0x63, 0xae, + 0x77, 0x7b, 0x66, 0x60, 0xc1, 0x3e, 0x80, 0x56, 0xb1, 0xc8, 0x0d, 0x16, + 0xde, 0x38, 0x82, 0x66, 0x99, 0x2b, 0x35, 0xd8, 0xb4, 0x5b, 0x4b, 0x3e, + 0x93, 0x96, 0x59, 0xf8, 0x96, 0x7e, 0x7b, 0x27, 0xf4, 0x62, 0xb7, 0xda, + 0x89, 0xa7, 0x34, 0x47, 0xed, 0xb3, 0x42, 0x20, 0xeb, 0xcd, 0xf6, 0xa3, + 0x9f, 0xf7, 0x48, 0x91, 0x17, 0xd2, 0x21, 0xed, 0x5a, 0x22, 0x39, 0xc9, + 0x76, 0x95, 0x36, 0xd9, 0x97, 0x0f, 0x19, 0xce, 0xd3, 0xbc, 0x74, 0x7d, + 0x53, 0x37, 0x3b, 0x4a, 0x97, 0xb7, 0xf8, 0x7e, 0xdd, 0x4c, 0x5f, 0xae, + 0x5c, 0x0b, 0xab, 0x4c, 0x34, 0xa1, 0x7e, 0x34, 0x35, 0xf4, 0xfc, 0x92, + 0xab, 0x2e, 0x6a, 0x15, 0xce, 0x84, 0xae, 0x70, 0xae, 0x85, 0x21, 0xe6, + 0x41, 0x13, 0x31, 0xe0, 0x8f, 0xab, 0x82, 0xe3, 0x09, 0xaf, 0xa4, 0x7c, + 0xb4, 0xb9, 0xb7, 0xc0, 0x67, 0x08, 0xc9, 0x9d, 0xcd, 0x0b, 0x3c, 0xa0, + 0x0c, 0xde, 0x49, 0x2f, 0x40, 0x19, 0x95, 0x64, 0xb9, 0x7c, 0x2a, 0x72, + 0xdd, 0xa2, 0x92, 0x0a, 0x21, 0xeb, 0x8c, 0xc3, 0x6d, 0x52, 0xe7, 0x05, + 0x50, 0x01, 0x55, 0x19, 0x2f, 0xbd, 0x1b, 0x72, 0x73, 0xfe, 0x82, 0x9f, + 0xbf, 0xa0, 0xfe, 0x19, 0x7c, 0x42, 0x6d, 0x76, 0x32, 0x47, 0x36, 0x15, + 0x2e, 0xde, 0xe8, 0xe6, 0xca, 0x07, 0xa3, 0x6b, 0x40, 0x99, 0x96, 0xcd, + 0x19, 0xea, 0x7e, 0xc9, 0x87, 0x9d, 0x3d, 0xa0, 0x82, 0x88, 0xe7, 0xe4, + 0x34, 0x9f, 0xa5, 0x27, 0xdf, 0xae, 0x03, 0x37, 0xa8, 0x35, 0x64, 0x02, + 0x09, 0x09, 0x9e, 0xec, 0x38, 0x0a, 0xff, 0x79, 0x8c, 0x9a, 0x87, 0x66, + 0xcd, 0xe4, 0xf4, 0x9d, 0xa9, 0x07, 0x96, 0x36, 0xae, 0x2e, 0x4e, 0xc5, + 0xe9, 0x86, 0xb2, 0x8e, 0x71, 0x5d, 0xe8, 0xee, 0x84, 0xf3, 0x30, 0x2a, + 0x58, 0x1a, 0x80, 0xb8, 0xaa, 0xb8, 0x1d, 0xc4, 0xae, 0x59, 0x91, 0xf3, + 0x16, 0x9b, 0xa3, 0x8a, 0xa3, 0x26, 0xb2, 0x0a, 0xe5, 0x58, 0xb7, 0x96, + 0x87, 0xfb, 0x00, 0xe4, 0x50, 0x7c, 0xb1, 0x77, 0x3a, 0x18, 0xc2, 0xe3, + 0xc1, 0x12, 0xa6, 0x0d, 0x06, 0xeb, 0x80, 0x6c, 0x5a, 0xee, 0x34, 0xcc, + 0x1c, 0x87, 0x35, 0x46, 0x1d, 0x05, 0x83, 0xd8, 0x91, 0x22, 0xaa, 0xf6, + 0xad, 0x87, 0xab, 0x76, 0x18, 0x79, 0xe2, 0x09, 0xc3, 0xa3, 0x15, 0x67, + 0x3a, 0x7c, 0x0f, 0xa0, 0x4c, 0x7b, 0xfc, 0xfc, 0xdd, 0x5c, 0xe4, 0x86, + 0x58, 0x13, 0xb8, 0x97, 0xae, 0x8c, 0x75, 0xc8, 0x02, 0x1e, 0x33, 0x45, + 0xa9, 0x54, 0x09, 0x15, 0x53, 0x4f, 0x28, 0x47, 0x4d, 0x5f, 0xd0, 0xc7, + 0x09, 0xbd, 0x93, 0xb0, 0x08, 0x79, 0x05, 0xbc, 0xbc, 0xaf, 0x2c, 0xbd, + 0xbb, 0x21, 0xd1, 0x60, 0xb8, 0x81, 0x4c, 0x6c, 0x5e, 0x45, 0x39, 0xa3, + 0x31, 0x54, 0xb7, 0x82, 0xef, 0x86, 0xe4, 0x5e, 0xca, 0xd6, 0xb8, 0x31, + 0xa2, 0x4c, 0x84, 0x5b, 0xac, 0xe5, 0x29, 0xbf, 0xbf, 0x89, 0xb4, 0x4c, + 0xd3, 0x69, 0x66, 0x50, 0xeb, 0xda, 0x7d, 0x00, 0xbb, 0x45, 0x0f, 0xe1, + 0xd1, 0x30, 0x1a, 0xc6, 0x94, 0x66, 0xdc, 0x01, 0x75, 0xce, 0xf8, 0xfc, + 0xd9, 0xce, 0xcf, 0x1f, 0x9e, 0x5a, 0x55, 0xa4, 0x3e, 0xe6, 0x51, 0xc7, + 0x74, 0x40, 0x82, 0x09, 0xea, 0xa0, 0xf5, 0xb2, 0x70, 0x9f, 0x0e, 0xfb, + 0x46, 0x8a, 0x69, 0xbf, 0x07, 0x92, 0xdc, 0x74, 0x03, 0x70, 0xc6, 0x44, + 0x81, 0x66, 0x40, 0xc7, 0xf5, 0xb8, 0xf0, 0x45, 0x0f, 0xca, 0xd8, 0xb0, + 0x9e, 0x48, 0x94, 0xff, 0x85, 0xcb, 0x7b, 0xec, 0x67, 0x5d, 0xfe, 0xe9, + 0x13, 0xd1, 0x67, 0x95, 0xd9, 0x35, 0x9e, 0x8a, 0x53, 0x4d, 0x6b, 0x9d, + 0x42, 0x53, 0xb1, 0x6b, 0x51, 0x1e, 0x35, 0x40, 0x81, 0x92, 0x91, 0x5f, + 0x1f, 0x8e, 0xbe, 0x37, 0xd3, 0x85, 0xab, 0x85, 0x37, 0x1c, 0x0f, 0xae, + 0xd9, 0xf7, 0xa2, 0x75, 0x3d, 0xd9, 0xd7, 0x2a, 0x80, 0xb0, 0x4c, 0x14, + 0x04, 0x40, 0xc5, 0xba, 0x0e, 0xbe, 0xab, 0xcc, 0x38, 0x35, 0x62, 0x6c, + 0xa5, 0xce, 0x49, 0x15, 0x2a, 0x10, 0xb5, 0x6a, 0xd2, 0x3b, 0xd2, 0x6a, + 0xad, 0x2e, 0x34, 0x46, 0x8b, 0x78, 0x57, 0x6e, 0xc4, 0xde, 0x65, 0x68, + 0x05, 0x8f, 0xd6, 0x6e, 0x34, 0xb9, 0xaa, 0x80, 0x77, 0xff, 0x6c, 0x1a, + 0x37, 0x87, 0xdd, 0x33, 0x13, 0x33, 0xa7, 0xa9, 0x3a, 0x90, 0x32, 0x7b, + 0x9b, 0x21, 0x31, 0xc8, 0xf5, 0x4c, 0xa6, 0x73, 0x42, 0x79, 0x46, 0x14, + 0x1b, 0xef, 0xf4, 0x78, 0xd9, 0x7e, 0x6f, 0x31, 0xaa, 0x59, 0x97, 0x34, + 0xe5, 0xe6, 0x67, 0xf3, 0x86, 0xf5, 0x61, 0xe7, 0x51, 0x6d, 0xce, 0xb3, + 0xdc, 0x86, 0xc7, 0x55, 0x43, 0xfa, 0x38, 0x78, 0xb0, 0x8d, 0x03, 0x9c, + 0xe4, 0x6c, 0xca, 0x73, 0x94, 0xa1, 0x0c, 0xb8, 0x11, 0xda, 0x0c, 0x0b, + 0x18, 0x1b, 0xd0, 0x99, 0xe7, 0xa9, 0x0d, 0xc3, 0x36, 0xd7, 0x8c, 0x16, + 0xad, 0x16, 0x1f, 0xb2, 0x3c, 0x07, 0x32, 0x11, 0x6c, 0xd2, 0x8f, 0x33, + 0x37, 0x5c, 0x3e, 0x4f, 0x7a, 0x76, 0xf7, 0x85, 0xcc, 0x68, 0x1a, 0xf9, + 0x26, 0x74, 0x42, 0xc9, 0xea, 0x21, 0x7e, 0x74, 0x3c, 0x4f, 0xde, 0xfb, + 0xd7, 0x83, 0x62, 0x12, 0xc7, 0x4f, 0xfc, 0x47, 0x18, 0x9d, 0xc5, 0xf5, + 0xe9, 0xd7, 0xaa, 0x76, 0x20, 0x99, 0x79, 0xae, 0x9b, 0x7a, 0xde, 0x8b, + 0x95, 0xc2, 0xa5, 0xa3, 0x6a, 0x30, 0x9b, 0x99, 0x63, 0x34, 0x7c, 0xd1, + 0x53, 0xa1, 0x6c, 0xd6, 0xed, 0x7d, 0x8c, 0xba, 0xc8, 0x21, 0xf3, 0xe1, + 0x31, 0x55, 0x3d, 0x88, 0x87, 0x04, 0xc7, 0xc9, 0x65, 0x0c, 0x53, 0x1e, + 0xd4, 0xd9, 0xaa, 0xda, 0xc2, 0x14, 0x88, 0xf2, 0x07, 0x2c, 0x12, 0x4d, + 0x79, 0x54, 0xaa, 0xd9, 0x47, 0x95, 0xf9, 0x7e, 0x26, 0x89, 0x4b, 0x63, + 0x7e, 0x44, 0x06, 0x0e, 0xe2, 0x8d, 0x9a, 0x0a, 0xc3, 0xee, 0x55, 0x13, + 0x55, 0x04, 0xcc, 0xb5, 0x2e, 0xa0, 0x0d, 0xec, 0x76, 0x84, 0xc1, 0x1e, + 0xdd, 0xe6, 0xfa, 0x54, 0x6e, 0x38, 0x30, 0x6f, 0xcc, 0xa4, 0x8d, 0x76, + 0x1e, 0xa3, 0x8e, 0x2c, 0x5e, 0x37, 0xeb, 0x0b, 0xf4, 0xb5, 0x80, 0xde, + 0x58, 0x13, 0x5a, 0x52, 0xdc, 0x65, 0x99, 0x1a, 0x1b, 0x75, 0x0c, 0xbd, + 0x83, 0xe8, 0x90, 0x8e, 0xa9, 0xbf, 0x42, 0x22, 0xe1, 0x3a, 0x31, 0x4e, + 0x54, 0xad, 0xd4, 0x6f, 0x80, 0xb4, 0xb5, 0x82, 0x05, 0x20, 0xd7, 0x38, + 0xd7, 0xeb, 0x25, 0x33, 0xe9, 0x4b, 0xc3, 0x5e, 0xd1, 0x11, 0xb0, 0xd9, + 0x8e, 0x90, 0x48, 0x2a, 0xe3, 0xa0, 0x60, 0x16, 0x70, 0xe3, 0xd1, 0x45, + 0x11, 0x64, 0x91, 0x69, 0x87, 0x1c, 0xbb, 0x91, 0xc4, 0x43, 0x12, 0x62, + 0x99, 0x69, 0xe5, 0x96, 0x01, 0x15, 0xdb, 0xdf, 0x05, 0x55, 0x34, 0xbb, + 0xd6, 0x76, 0x89, 0xcd, 0xb5, 0x4f, 0x2e, 0xa7, 0x6e, 0x15, 0xc9, 0xc0, + 0x8e, 0xa8, 0x63, 0x79, 0x12, 0xfb, 0x7e, 0x69, 0x8f, 0x52, 0x5e, 0xe7, + 0x76, 0x16, 0x28, 0x76, 0xca, 0xcb, 0xd8, 0x0e, 0x4a, 0x93, 0x9d, 0x16, + 0x68, 0x98, 0xf8, 0xc3, 0x39, 0xb2, 0x2d, 0xea, 0xba, 0x72, 0x16, 0x33, + 0xb7, 0xec, 0x61, 0x9e, 0x94, 0x32, 0x01, 0x22, 0xde, 0x66, 0xfd, 0x68, + 0xfa, 0xcf, 0xf2, 0x52, 0x4f, 0x02, 0xe8, 0x25, 0xd3, 0xa3, 0x5b, 0x29, + 0xae, 0xe9, 0x62, 0xfa, 0xd6, 0x1a, 0x50, 0x80, 0x95, 0x96, 0xdf, 0x00, + 0xfc, 0x23, 0xf1, 0x95, 0xef, 0xbb, 0xf5, 0x23, 0x9d, 0x6b, 0xd6, 0xed, + 0xb4, 0xe2, 0x4a, 0xf6, 0xb8, 0x20, 0x83, 0x6b, 0x45, 0x92, 0x29, 0x5a, + 0x02, 0xe9, 0xf7, 0x8e, 0x5c, 0x02, 0xde, 0xb4, 0x9a, 0xdf, 0x18, 0x10, + 0x17, 0x7f, 0xd8, 0x2e, 0x17, 0xc0, 0xf0, 0x6b, 0x3b, 0x88, 0x09, 0x58, + 0xf2, 0x18, 0x22, 0x09, 0x80, 0x4a, 0xe0, 0x51, 0x6f, 0x7a, 0x70, 0x09, + 0x1f, 0xe5, 0xfa, 0xa9, 0x4d, 0x24, 0x1f, 0x18, 0x1c, 0x74, 0xcd, 0x87, + 0x04, 0xfd, 0x85, 0x33, 0x4c, 0x28, 0xbd, 0xa3, 0x66, 0x6c, 0x99, 0x7e, + 0x50, 0x5e, 0xb5, 0x22, 0x33, 0x92, 0xd4, 0xd8, 0x82, 0x4e, 0x38, 0xbe, + 0xcb, 0x3d, 0x5f, 0x19, 0xd1, 0x0f, 0x8b, 0xa1, 0x78, 0x08, 0x1c, 0x10, + 0x0b, 0x77, 0xa7, 0x39, 0x2e, 0x91, 0x83, 0xee, 0x1d, 0x36, 0xd8, 0x77, + 0x87, 0x8a, 0x38, 0x45, 0x3c, 0xbd, 0xb9, 0x88, 0xbb, 0x1b, 0x20, 0xd1, + 0x95, 0xb9, 0x8f, 0x03, 0x46, 0xfa, 0xab, 0x70, 0x68, 0x26, 0xd9, 0xb1, + 0x25, 0x52, 0x5a, 0x77, 0x2d, 0x92, 0xc2, 0x1d, 0xb6, 0x6e, 0xec, 0x67, + 0xef, 0x34, 0xe2, 0x64, 0xb3, 0xa0, 0xae, 0x0c, 0xd9, 0x36, 0xa1, 0xc7, + 0xd8, 0xbf, 0x7a, 0x43, 0xbf, 0xc0, 0xc6, 0x90, 0x60, 0x6a, 0x23, 0xc0, + 0x6a, 0x5d, 0x62, 0x18, 0xac, 0xc1, 0x20, 0x35, 0x17, 0xba, 0x4e, 0x54, + 0xb7, 0xec, 0xd4, 0xad, 0x99, 0x94, 0xa4, 0xda, 0x57, 0xe7, 0x46, 0xed, + 0x47, 0xd1, 0xb4, 0xa2, 0x3e, 0x0f, 0x4a, 0xb6, 0xa6, 0x68, 0x3e, 0x94, + 0xb9, 0x18, 0x30, 0xe0, 0x75, 0x08, 0xe8, 0xf3, 0x21, 0x79, 0x26, 0x68, + 0x6a, 0x65, 0xb6, 0xbe, 0x03, 0x98, 0x8f, 0x04, 0xad, 0x1e, 0xb0, 0x54, + 0xd2, 0x28, 0xdd, 0x4a, 0xe9, 0xf3, 0xa0, 0x06, 0xbf, 0x0b, 0x2a, 0xee, + 0xf8, 0x03, 0x7e, 0x1d, 0x37, 0xc1, 0x32, 0xd1, 0x41, 0xf4, 0x9b, 0xc5, + 0x02, 0x10, 0x6f, 0x55, 0x5a, 0xec, 0x5b, 0xe7, 0x61, 0x05, 0x17, 0xf0, + 0xf8, 0xc6, 0x89, 0xe8, 0xad, 0x32, 0x57, 0x14, 0xe5, 0xf8, 0xf5, 0x88, + 0xd9, 0x73, 0x17, 0x10, 0xa7, 0xc3, 0xf8, 0x78, 0x0b, 0x66, 0xab, 0x63, + 0x4f, 0x96, 0x5d, 0xdf, 0x36, 0x83, 0xc4, 0x6f, 0x20, 0xbd, 0xcb, 0x4c, + 0xd2, 0xfa, 0x35, 0x87, 0xd8, 0xb6, 0xbb, 0xcc, 0xb6, 0xd2, 0x85, 0x03, + 0x6a, 0xea, 0xbb, 0x6d, 0x2f, 0xa2, 0x06, 0xc0, 0xd6, 0x68, 0xd9, 0x7f, + 0xd6, 0xa2, 0x3b, 0x08, 0x6a, 0x98, 0x26, 0x6d, 0x9a, 0x2b, 0x68, 0x51, + 0x78, 0xde, 0xa6, 0x96, 0x50, 0x7b, 0xfc, 0x03, 0x43, 0xf8, 0x21, 0x01, + 0x9d, 0xe2, 0x89, 0x65, 0x47, 0xae, 0x9c, 0x45, 0x5e, 0xa5, 0xce, 0x97, + 0xb3, 0xe6, 0xf6, 0xd4, 0x5a, 0xe8, 0x6b, 0x87, 0xd6, 0xdf, 0xfb, 0x1f, + 0xaf, 0xfb, 0xaf, 0x19, 0xa5, 0xfd, 0xba, 0xe0, 0x22, 0x2f, 0x91, 0x97, + 0xdf, 0xae, 0xe9, 0x39, 0xb1, 0xe4, 0xd3, 0x10, 0xcb, 0xb3, 0x03, 0xb5, + 0x0b, 0xf0, 0xd9, 0x70, 0x1e, 0x9c, 0x63, 0x6f, 0x3a, 0xcf, 0x3c, 0x1b, + 0x86, 0xa3, 0xad, 0x1a, 0xe7, 0x4c, 0x09, 0xd0, 0x80, 0xf6, 0x8b, 0x72, + 0x96, 0x53, 0x7e, 0x66, 0xfb, 0x7c, 0x7c, 0x8a, 0xb0, 0x60, 0xa6, 0x4c, + 0x20, 0xc4, 0x63, 0x69, 0x6a, 0xc3, 0x53, 0xf8, 0x9a, 0x28, 0x30, 0x9d, + 0x6f, 0x0e, 0x1b, 0xb2, 0x2c, 0xe6, 0x94, 0x9f, 0xfc, 0xc0, 0x8d, 0x71, + 0xbe, 0x37, 0xa6, 0xc9, 0xbd, 0x3c, 0x4a, 0xf3, 0xc4, 0xb3, 0x88, 0x4c, + 0x45, 0x26, 0x4e, 0x2f, 0x83, 0x16, 0x70, 0xb6, 0xc7, 0xb2, 0x36, 0xf0, + 0x0c, 0x67, 0xd2, 0x0a, 0xd3, 0xd9, 0x7c, 0x35, 0x29, 0xac, 0xd4, 0x9c, + 0x6d, 0xfc, 0xec, 0x58, 0x92, 0xf0, 0xba, 0x32, 0x00, 0xae, 0xb1, 0xeb, + 0x4d, 0x8c, 0x1a, 0x20, 0xe7, 0x5c, 0xfc, 0x9a, 0x4d, 0x51, 0x24, 0x7b, + 0x52, 0xeb, 0x13, 0x3d, 0xb4, 0xab, 0xda, 0xb3, 0x74, 0x39, 0xd2, 0xf8, + 0x2d, 0xef, 0x9b, 0x0f, 0xae, 0xf5, 0x3c, 0x99, 0x34, 0xbe, 0x15, 0x5c, + 0x9f, 0x5d, 0xae, 0xf4, 0x72, 0xc2, 0xac, 0x06, 0xbe, 0xad, 0xe4, 0x68, + 0xea, 0xd5, 0xa1, 0xdc, 0xdb, 0xf4, 0x61, 0x51, 0xf5, 0x1a, 0x62, 0x15, + 0xfd, 0x00, 0x51, 0x35, 0x53, 0x6c, 0x39, 0x3e, 0xdb, 0x60, 0x0a, 0x52, + 0xc1, 0x52, 0x3c, 0xd7, 0xab, 0x73, 0xea, 0x1e, 0x38, 0x38, 0x65, 0x35, + 0x35, 0x2b, 0x28, 0x04, 0x5c, 0x82, 0xea, 0x4a, 0x9e, 0x96, 0x72, 0xa4, + 0x8e, 0x42, 0xfd, 0x55, 0xa8, 0x66, 0x7a, 0x40, 0xc9, 0xf2, 0xc2, 0x1e, + 0x5d, 0x09, 0x90, 0x32, 0x18, 0xdb, 0x11, 0x4c, 0x6c, 0x9c, 0x27, 0x62, + 0x0a, 0xe6, 0xc1, 0xdf, 0xf2, 0x6a, 0x8c, 0x26, 0xb4, 0xfb, 0xda, 0xa9, + 0x08, 0x10, 0x3a, 0xf0, 0xe1, 0x64, 0xe5, 0x03, 0x81, 0x7d, 0x15, 0x74, + 0xa1, 0x8d, 0x10, 0xc8, 0xbb, 0x6a, 0x7c, 0x60, 0xa1, 0x09, 0x35, 0x19, + 0x2d, 0x70, 0xb5, 0x36, 0xc8, 0x8b, 0x66, 0x5f, 0xe0, 0xe7, 0xea, 0x70, + 0x2f, 0x5d, 0x3f, 0xae, 0x5e, 0x25, 0x84, 0xdd, 0x9b, 0x69, 0x44, 0x37, + 0x7c, 0x6b, 0x9e, 0x81, 0x18, 0x36, 0x4b, 0xff, 0x86, 0x44, 0x2a, 0x39, + 0x66, 0x7f, 0x71, 0x43, 0xe7, 0x65, 0xfe, 0xfd, 0x34, 0xb9, 0xd9, 0x5a, + 0x00, 0xd1, 0x41, 0x43, 0xc7, 0xbc, 0x65, 0x68, 0xb7, 0x73, 0xff, 0x19, + 0xd3, 0xed, 0x15, 0xa4, 0x67, 0xa1, 0x53, 0x0e, 0xa6, 0xfb, 0x25, 0xce, + 0x9d, 0x5b, 0x73, 0x08, 0xf3, 0x3b, 0x69, 0xe4, 0x94, 0x9b, 0x94, 0x03, + 0xb3, 0x8a, 0x2e, 0x07, 0x0c, 0xef, 0x18, 0x4c, 0x2b, 0x1c, 0x83, 0x9f, + 0x25, 0x20, 0x29, 0x72, 0x11, 0xa0, 0xaa, 0xed, 0x0c, 0xf9, 0xce, 0x94, + 0x0d, 0x7a, 0xb6, 0xb3, 0xa4, 0x57, 0xd6, 0x61, 0xca, 0x1a, 0x0e, 0x89, + 0x6d, 0x99, 0x4d, 0x06, 0xcd, 0x83, 0x7e, 0x09, 0x14, 0x5b, 0xe7, 0x4c, + 0x72, 0xa8, 0x98, 0xc8, 0x27, 0xf3, 0x70, 0x89, 0x87, 0x11, 0xbb, 0x98, + 0x82, 0x77, 0x9d, 0xaa, 0x95, 0x8c, 0xc1, 0xf8, 0x39, 0x27, 0xd5, 0x64, + 0x59, 0x6a, 0x8c, 0xbe, 0xe2, 0xe1, 0xd1, 0x6b, 0xe3, 0xaf, 0x30, 0x6f, + 0xf4, 0x9e, 0x35, 0x0b, 0x10, 0x24, 0x77, 0xd8, 0xa4, 0x30, 0x2e, 0xf7, + 0x97, 0xfd, 0xef, 0x1e, 0x9e, 0xf2, 0xbd, 0xf2, 0x41, 0x73, 0x19, 0xe6, + 0x7b, 0x7f, 0x74, 0x11, 0x91, 0x38, 0xc5, 0xac, 0xd5, 0xb0, 0x48, 0xc4, + 0xe9, 0x41, 0xd4, 0x50, 0x76, 0x13, 0xbf, 0xec, 0xe8, 0x3a, 0xa8, 0x84, + 0x42, 0x98, 0x12, 0x64, 0x95, 0x85, 0x79, 0x29, 0xea, 0x3a, 0xf9, 0xa4, + 0x5c, 0x9c, 0x35, 0x01, 0x68, 0x71, 0xb9, 0x5b, 0xbe, 0xaa, 0x76, 0x9e, + 0x63, 0x1c, 0xc1, 0x83, 0x94, 0xc6, 0x89, 0x2b, 0x1d, 0x00, 0x43, 0x74, + 0x00, 0x41, 0x93, 0x58, 0x52, 0xf9, 0x13, 0xfe, 0x9f, 0x7a, 0xb7, 0x3d, + 0x6b, 0x70, 0x4e, 0x4f, 0x8f, 0xf4, 0x9c, 0xe4, 0x97, 0x62, 0xaf, 0x69, + 0x45, 0xec, 0xf4, 0x53, 0x71, 0xdc, 0xc7, 0x8d, 0x6f, 0xb2, 0x9d, 0xec, + 0x43, 0xdd, 0xc0, 0xe5, 0xd1, 0x6c, 0x1a, 0x82, 0x19, 0xf6, 0x18, 0xd3, + 0x59, 0x0e, 0x07, 0x81, 0x5a, 0x23, 0x10, 0x8b, 0xaa, 0x0b, 0x99, 0xc8, + 0x34, 0xc2, 0xd0, 0xa9, 0x69, 0x7f, 0x54, 0xe3, 0xc4, 0xa0, 0xe7, 0x4b, + 0x31, 0x90, 0xe7, 0x3b, 0x45, 0x9b, 0x7f, 0xae, 0xd2, 0xab, 0x22, 0xb9, + 0xfc, 0x07, 0x39, 0x4b, 0x45, 0x83, 0x8d, 0x41, 0x7a, 0x52, 0xb2, 0xae, + 0x71, 0x78, 0x17, 0x63, 0xfa, 0xbe, 0x59, 0xca, 0xf0, 0xfd, 0x68, 0xe5, + 0xc4, 0x9a, 0x74, 0x3d, 0xec, 0xd4, 0x8b, 0xa1, 0x2c, 0x31, 0x4d, 0x73, + 0xfd, 0x5c, 0x1e, 0xeb, 0x5f, 0xf6, 0x42, 0x0d, 0x79, 0x5f, 0x64, 0x10, + 0xae, 0xb2, 0xf6, 0x9e, 0xa8, 0xab, 0xa5, 0x2b, 0x9a, 0xcf, 0x25, 0xfa, + 0xa2, 0xb3, 0xdc, 0x30, 0x3d, 0x08, 0x4e, 0xbb, 0x7b, 0x0c, 0x28, 0x34, + 0x9d, 0xda, 0xc4, 0x94, 0xa4, 0xf4, 0x1e, 0x78, 0x8b, 0xa9, 0xd3, 0xa7, + 0x1c, 0x2a, 0x27, 0x14, 0xa0, 0x44, 0x1a, 0x9a, 0x87, 0x72, 0xa5, 0x6d, + 0x69, 0x46, 0xe5, 0xc1, 0x4f, 0x29, 0x87, 0xc0, 0xa7, 0xa8, 0x96, 0xde, + 0xa9, 0x63, 0x08, 0xd8, 0x4a, 0xa1, 0x25, 0x43, 0x76, 0x41, 0xf7, 0x9f, + 0x17, 0xe3, 0xe1, 0x4b, 0xc6, 0x2b, 0x79, 0xea, 0xd5, 0xa7, 0x72, 0x16, + 0x0a, 0x8c, 0xcd, 0x49, 0x70, 0x75, 0xd4, 0x59, 0x4a, 0x19, 0x7b, 0x31, + 0x02, 0x7a, 0x3a, 0x20, 0x15, 0x62, 0x7e, 0x4e, 0x6f, 0xac, 0xd0, 0xd1, + 0x29, 0xbd, 0x2d, 0xa1, 0xc6, 0x3e, 0xa6, 0x1a, 0x26, 0x18, 0x96, 0x98, + 0x12, 0x56, 0x37, 0xbf, 0xb4, 0x91, 0x57, 0xe8, 0xda, 0x61, 0x7c, 0x2f, + 0x3e, 0xd4, 0x51, 0xfe, 0xe8, 0x5b, 0x00, 0x30, 0x08, 0xf6, 0x4e, 0x69, + 0xa8, 0x1a, 0x2b, 0x82, 0x41, 0x85, 0xa9, 0xd9, 0x3c, 0xc8, 0x02, 0x91, + 0x99, 0xd4, 0xa2, 0xfd, 0x9d, 0x1b, 0x08, 0xfc, 0x41, 0x3e, 0x10, 0x6b, + 0x80, 0x74, 0x3d, 0x72, 0x61, 0x97, 0xdd, 0x96, 0xec, 0xf4, 0xd6, 0x6d, + 0x68, 0x02, 0x6e, 0xbb, 0x55, 0x9d, 0x6f, 0x11, 0xde, 0xd1, 0xad, 0x6d, + 0x42, 0x96, 0x2c, 0x42, 0x1e, 0xa9, 0x19, 0x42, 0x22, 0x38, 0x38, 0x18, + 0x3c, 0x4b, 0xc1, 0x9c, 0x0f, 0xe1, 0x34, 0x61, 0x06, 0x77, 0x54, 0x04, + 0xe0, 0x87, 0x94, 0x5c, 0xc9, 0xa1, 0x35, 0x55, 0x3d, 0x4a, 0xf2, 0x4f, + 0x05, 0x11, 0x98, 0x6f, 0x3c, 0x85, 0x84, 0xe6, 0xf8, 0x71, 0x8a, 0xdf, + 0xe9, 0x9a, 0xe3, 0x70, 0xd6, 0x36, 0xd6, 0xc8, 0x66, 0x3e, 0xba, 0x7c, + 0x0a, 0x23, 0x0a, 0xd0, 0xb6, 0x66, 0x68, 0xa8, 0xdf, 0x37, 0x17, 0xfb, + 0xdd, 0x9c, 0x8b, 0xc7, 0x8e, 0xc4, 0x4f, 0x40, 0x08, 0x23, 0x58, 0x15, + 0xa2, 0xba, 0xef, 0xdf, 0x67, 0xcd, 0x1f, 0xb6, 0xc4, 0xea, 0xce, 0x81, + 0x38, 0x58, 0x92, 0x57, 0xcf, 0x83, 0x47, 0x29, 0x9f, 0xde, 0x9b, 0xde, + 0x01, 0xfe, 0x68, 0x91, 0x67, 0x06, 0x9d, 0x31, 0xd0, 0xb9, 0xc3, 0xbb, + 0xc3, 0x6b, 0xa0, 0x04, 0x1e, 0x34, 0xd5, 0x38, 0xd4, 0xac, 0x70, 0xae, + 0xab, 0xb2, 0xbd, 0x4b, 0xa0, 0xad, 0x2b, 0x82, 0xaf, 0x8c, 0x90, 0x4d, + 0xd3, 0xca, 0x71, 0x35, 0x75, 0x89, 0xe5, 0x42, 0x91, 0x46, 0x8d, 0x18, + 0x04, 0x7a, 0xb9, 0xaa, 0x3b, 0xe7, 0x1e, 0x8c, 0x4e, 0xf9, 0x6e, 0x74, + 0xaa, 0x2e, 0x36, 0x86, 0xfb, 0xef, 0x9c, 0xd7, 0xba, 0x5e, 0x2e, 0x3c, + 0x40, 0xce, 0x8b, 0x2b, 0x94, 0x55, 0xf2, 0xd4, 0x7d, 0xbf, 0x8c, 0x8a, + 0xa8, 0x59, 0x84, 0x6f, 0x32, 0x95, 0xc5, 0xcc, 0xad, 0xee, 0x30, 0x23, + 0x7c, 0x54, 0xea, 0x60, 0xb8, 0x88, 0x12, 0x45, 0x03, 0xbc, 0xe3, 0x92, + 0x9f, 0xa8, 0x5b, 0x07, 0x97, 0x53, 0x0d, 0xe1, 0xe3, 0x3d, 0xdf, 0xf2, + 0x2a, 0x12, 0xee, 0xdf, 0x73, 0x8d, 0x41, 0xf4, 0xe4, 0x2c, 0xb4, 0xd4, + 0x9e, 0xfe, 0xf2, 0xe6, 0xa0, 0x9e, 0x2a, 0x3a, 0x36, 0x26, 0x7e, 0xd9, + 0xe1, 0x22, 0xee, 0x0b, 0x5b, 0x48, 0xd2, 0xa9, 0x55, 0xab, 0x50, 0x7c, + 0xf6, 0xc8, 0x56, 0x31, 0xbb, 0x51, 0xe9, 0x31, 0x4d, 0xaa, 0x13, 0x3a, + 0x99, 0x9f, 0x8c, 0x59, 0x6a, 0xc9, 0xf1, 0x0a, 0x89, 0xcc, 0x39, 0x98, + 0xbd, 0xc3, 0x93, 0x97, 0x28, 0xe5, 0x73, 0x94, 0xf2, 0x0a, 0x7a, 0x09, + 0x38, 0x0b, 0xab, 0xd8, 0x49, 0x98, 0x14, 0x34, 0x32, 0x9d, 0xef, 0x9d, + 0x47, 0xdb, 0x82, 0xb9, 0x84, 0xd6, 0xd7, 0x9f, 0xf7, 0xdf, 0x79, 0x5b, + 0xe8, 0x92, 0x44, 0x31, 0x5d, 0x42, 0x80, 0x90, 0x8d, 0x36, 0xa2, 0x39, + 0x02, 0x64, 0x21, 0xa2, 0xb8, 0xfc, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x4c, 0xe9, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0xd8, 0x03, 0x00, 0x00, 0xdc, 0x03, 0x00, 0x00, 0xe0, 0x03, 0x00, 0x00, + 0x0f, 0x00, 0x00, 0x00, 0xa8, 0x03, 0x00, 0x00, 0x50, 0x03, 0x00, 0x00, + 0x04, 0x03, 0x00, 0x00, 0xac, 0x02, 0x00, 0x00, 0x74, 0x02, 0x00, 0x00, + 0x2c, 0x02, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, 0xac, 0x01, 0x00, 0x00, + 0x74, 0x01, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, + 0x9c, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x9e, 0xfc, 0xff, 0xff, 0x0c, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x35, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, + 0x5e, 0xfd, 0xff, 0xff, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, + 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x00, 0x08, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x3f, 0x01, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x96, 0xfd, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x88, 0xfd, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x2f, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, + 0xca, 0xfd, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + 0x18, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x78, 0xfd, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, + 0x2d, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x0e, 0xfe, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x18, 0x00, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xbc, 0xfd, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, + 0x26, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, + 0x29, 0x00, 0x00, 0x00, 0x52, 0xfe, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x06, 0x18, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, + 0x96, 0xfe, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x10, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x88, 0xfe, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, + 0x1f, 0x00, 0x00, 0x00, 0xca, 0xfe, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x06, 0x18, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x78, 0xfe, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, + 0x1a, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x0e, 0xff, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x10, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, + 0x17, 0x00, 0x00, 0x00, 0x42, 0xff, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x06, 0x18, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xf0, 0xfe, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x86, 0xff, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x10, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x78, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x0f, 0x00, 0x00, 0x00, 0xba, 0xff, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x06, 0x18, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x68, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x18, 0x00, 0x14, 0x00, 0x10, 0x00, 0x0c, 0x00, + 0x0b, 0x00, 0x04, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x14, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x0c, 0x00, + 0x0b, 0x00, 0x04, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x06, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x07, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x10, 0x00, 0x0c, 0x00, + 0x08, 0x00, 0x04, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x34, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0xac, 0x12, 0x00, 0x00, + 0x3c, 0x12, 0x00, 0x00, 0xdc, 0x11, 0x00, 0x00, 0x90, 0x11, 0x00, 0x00, + 0x24, 0x11, 0x00, 0x00, 0xac, 0x10, 0x00, 0x00, 0x5c, 0x10, 0x00, 0x00, + 0x10, 0x10, 0x00, 0x00, 0xa8, 0x0f, 0x00, 0x00, 0x58, 0x0f, 0x00, 0x00, + 0x04, 0x0f, 0x00, 0x00, 0xb8, 0x0e, 0x00, 0x00, 0x4c, 0x0e, 0x00, 0x00, + 0xe4, 0x0d, 0x00, 0x00, 0x94, 0x0d, 0x00, 0x00, 0x48, 0x0d, 0x00, 0x00, + 0xe0, 0x0c, 0x00, 0x00, 0x90, 0x0c, 0x00, 0x00, 0x3c, 0x0c, 0x00, 0x00, + 0xf0, 0x0b, 0x00, 0x00, 0x84, 0x0b, 0x00, 0x00, 0x1c, 0x0b, 0x00, 0x00, + 0xcc, 0x0a, 0x00, 0x00, 0x80, 0x0a, 0x00, 0x00, 0x18, 0x0a, 0x00, 0x00, + 0xc8, 0x09, 0x00, 0x00, 0x74, 0x09, 0x00, 0x00, 0x28, 0x09, 0x00, 0x00, + 0xbc, 0x08, 0x00, 0x00, 0x54, 0x08, 0x00, 0x00, 0x04, 0x08, 0x00, 0x00, + 0xb8, 0x07, 0x00, 0x00, 0x50, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, + 0xac, 0x06, 0x00, 0x00, 0x60, 0x06, 0x00, 0x00, 0xf4, 0x05, 0x00, 0x00, + 0x8c, 0x05, 0x00, 0x00, 0x3c, 0x05, 0x00, 0x00, 0xe8, 0x04, 0x00, 0x00, + 0x9c, 0x04, 0x00, 0x00, 0x30, 0x04, 0x00, 0x00, 0xc8, 0x03, 0x00, 0x00, + 0x78, 0x03, 0x00, 0x00, 0x24, 0x03, 0x00, 0x00, 0xd8, 0x02, 0x00, 0x00, + 0x6c, 0x02, 0x00, 0x00, 0x04, 0x02, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, + 0x68, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, + 0x50, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x3a, 0xee, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x2c, 0x00, 0x00, 0x00, 0x94, 0xee, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x82, 0xee, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x2c, 0x00, 0x00, 0x00, 0xdc, 0xee, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0xd7, 0x23, 0x3a, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0xca, 0xee, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x44, 0x00, 0x00, 0x00, 0xbc, 0xee, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x00, 0x80, 0x37, 0x01, 0x00, 0x00, 0x00, + 0xc2, 0xff, 0x7f, 0x3f, 0x01, 0x00, 0x00, 0x00, 0xd2, 0x6f, 0x75, 0x36, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x2a, 0xef, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, + 0x48, 0x00, 0x00, 0x00, 0x1c, 0xef, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, 0x16, 0x49, 0x3d, + 0x01, 0x00, 0x00, 0x00, 0x87, 0x19, 0xb1, 0x40, 0x01, 0x00, 0x00, 0x00, + 0x58, 0x80, 0xdf, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0xfa, 0xef, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x2c, 0x00, 0x00, 0x00, + 0xec, 0xef, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x5d, 0xd1, 0xce, 0x39, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x42, 0xf0, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, + 0x34, 0xf0, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x23, 0x20, 0xb6, 0x3b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x22, 0xf0, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, + 0x14, 0xf0, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xa2, 0x5a, 0x91, 0x3d, 0x01, 0x00, 0x00, 0x00, + 0x47, 0xc9, 0x90, 0x41, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0xf2, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x48, 0x00, 0x00, 0x00, 0x7c, 0xf0, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x81, 0xb7, 0xf1, 0x39, 0x01, 0x00, 0x00, 0x00, 0x9e, 0xb5, 0x71, 0x41, + 0x01, 0x00, 0x00, 0x00, 0x33, 0x20, 0x70, 0xc1, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x5a, 0xf1, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x2c, 0x00, 0x00, 0x00, 0x4c, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x7a, 0x08, 0x97, 0x35, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0xa2, 0xf1, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x34, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x30, 0x00, 0x00, 0x00, 0x94, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x2f, 0xf5, 0x1f, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0xf2, 0xf1, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, + 0xe4, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xc7, 0xea, 0x1a, 0x3c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0xd2, 0xf1, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, + 0xc4, 0xf1, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xb2, 0x78, 0x3f, 0x3d, 0x01, 0x00, 0x00, 0x00, + 0x39, 0xb9, 0x3e, 0x41, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xb0, 0xf3, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x48, 0x00, 0x00, 0x00, 0x2c, 0xf2, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x89, 0x25, 0xf2, 0x39, 0x01, 0x00, 0x00, 0x00, 0xde, 0xdc, 0x1d, 0x41, + 0x01, 0x00, 0x00, 0x00, 0xa5, 0x23, 0x72, 0xc1, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x0a, 0xf3, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x2c, 0x00, 0x00, 0x00, 0xfc, 0xf2, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x42, 0xe0, 0x90, 0x35, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x52, 0xf3, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x34, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x30, 0x00, 0x00, 0x00, 0x44, 0xf3, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x1a, 0x2a, 0x19, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0xa2, 0xf3, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, + 0x94, 0xf3, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xe9, 0x36, 0xdd, 0x3b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x82, 0xf3, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, + 0x74, 0xf3, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xdd, 0x43, 0x7e, 0x3d, 0x01, 0x00, 0x00, 0x00, + 0x99, 0x45, 0x7d, 0x41, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x60, 0xf5, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x48, 0x00, 0x00, 0x00, 0xdc, 0xf3, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x5c, 0xfd, 0xa9, 0x39, 0x01, 0x00, 0x00, 0x00, 0x1e, 0xaa, 0x87, 0x40, + 0x01, 0x00, 0x00, 0x00, 0x08, 0xfc, 0x29, 0xc1, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0xba, 0xf4, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x2c, 0x00, 0x00, 0x00, 0xac, 0xf4, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x55, 0xf7, 0x52, 0x35, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x02, 0xf5, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x34, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x30, 0x00, 0x00, 0x00, 0xf4, 0xf4, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xd0, 0xda, 0x1e, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x52, 0xf5, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, + 0x44, 0xf5, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x8e, 0x0b, 0xa8, 0x3b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x32, 0xf5, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, + 0x24, 0xf5, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xf5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x12, 0x1c, 0x6e, 0x3d, 0x01, 0x00, 0x00, 0x00, + 0xdd, 0x4a, 0x00, 0x41, 0x01, 0x00, 0x00, 0x00, 0x31, 0xc6, 0xd9, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0xf6, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x2c, 0x00, 0x00, 0x00, 0xf4, 0xf5, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0x9d, 0x16, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x4a, 0xf6, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, 0x3c, 0xf6, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa4, 0x34, 0xab, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x2a, 0xf6, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, 0x1c, 0xf6, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x2e, 0x36, 0xe1, 0x3c, 0x01, 0x00, 0x00, 0x00, 0xf8, 0x54, 0xe0, 0x40, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x08, 0xf8, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x48, 0x00, 0x00, 0x00, 0x84, 0xf6, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe1, 0xd0, 0xa2, 0x39, + 0x01, 0x00, 0x00, 0x00, 0x9b, 0xcf, 0x22, 0x41, 0x01, 0x00, 0x00, 0x00, + 0xea, 0x23, 0x12, 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, + 0x62, 0xf7, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x2c, 0x00, 0x00, 0x00, + 0x54, 0xf7, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x99, 0xd3, 0xf7, 0x34, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0xaa, 0xf7, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x30, 0x00, 0x00, 0x00, + 0x9c, 0xf7, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0xd5, 0xc2, 0x3a, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xfa, 0xf7, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, 0xec, 0xf7, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x8f, 0x84, 0xa2, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xda, 0xf7, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, 0xcc, 0xf7, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xf7, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x64, 0xeb, 0x8e, 0x3d, 0x01, 0x00, 0x00, 0x00, 0x3b, 0xf3, 0x17, 0x41, + 0x01, 0x00, 0x00, 0x00, 0xb7, 0xc5, 0x04, 0xc1, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0xaa, 0xf8, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x2c, 0x00, 0x00, 0x00, 0x9c, 0xf8, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x92, 0xa8, 0x98, 0x39, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0xf2, 0xf8, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, + 0x2c, 0x00, 0x00, 0x00, 0xe4, 0xf8, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x58, 0x76, 0xb9, 0x3b, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xd2, 0xf8, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, + 0x48, 0x00, 0x00, 0x00, 0xc4, 0xf8, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x43, 0xb8, 0x52, 0x3d, + 0x01, 0x00, 0x00, 0x00, 0x8b, 0xe5, 0x51, 0x41, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0xb0, 0xfa, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x48, 0x00, 0x00, 0x00, + 0x2c, 0xf9, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xe3, 0xa1, 0xf0, 0x39, 0x01, 0x00, 0x00, 0x00, + 0x02, 0xa0, 0x70, 0x41, 0x01, 0x00, 0x00, 0x00, 0x87, 0x08, 0x65, 0xc1, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x0a, 0xfa, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x2c, 0x00, 0x00, 0x00, 0xfc, 0xf9, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xcc, 0x98, 0x41, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x52, 0xfa, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x30, 0x00, 0x00, 0x00, 0x44, 0xfa, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xed, 0xf5, 0xcd, 0x3a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0xa2, 0xfa, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, + 0x2c, 0x00, 0x00, 0x00, 0x94, 0xfa, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x9d, 0xca, 0xd4, 0x3b, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x82, 0xfa, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, + 0x48, 0x00, 0x00, 0x00, 0x74, 0xfa, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x58, 0x58, 0xce, 0x3d, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x49, 0x41, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x06, 0x52, 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x52, 0xfb, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x2c, 0x00, 0x00, 0x00, + 0x44, 0xfb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x9b, 0x9c, 0xe1, 0x39, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x9a, 0xfb, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, + 0x8c, 0xfb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xf8, 0xb6, 0xc3, 0x3b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x7a, 0xfb, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, + 0x6c, 0xfb, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x94, 0x8d, 0x93, 0x3d, 0x01, 0x00, 0x00, 0x00, + 0x06, 0xfa, 0x92, 0x41, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x58, 0xfd, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x48, 0x00, 0x00, 0x00, 0xd4, 0xfb, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x7a, 0xf6, 0x5f, 0x3a, 0x01, 0x00, 0x00, 0x00, 0xba, 0xf4, 0xdf, 0x41, + 0x01, 0x00, 0x00, 0x00, 0xf4, 0x7c, 0xcf, 0xc1, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x00, 0xb2, 0xfc, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x2c, 0x00, 0x00, 0x00, 0xa4, 0xfc, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x46, 0x2f, 0xc4, 0x35, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0xfa, 0xfc, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x34, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x30, 0x00, 0x00, 0x00, 0xec, 0xfc, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x8f, 0x3f, 0xe0, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x4a, 0xfd, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, + 0x3c, 0xfd, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x25, 0xd7, 0xa9, 0x3b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x2a, 0xfd, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, + 0x1c, 0xfd, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xc4, 0xf4, 0x39, 0x3e, 0x01, 0x00, 0x00, 0x00, + 0xf4, 0x1f, 0xe3, 0x41, 0x01, 0x00, 0x00, 0x00, 0xaa, 0x55, 0x8f, 0xc1, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xfa, 0xfd, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x2c, 0x00, 0x00, 0x00, 0xec, 0xfd, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x8b, 0x00, 0x4b, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x42, 0xfe, 0xff, 0xff, + 0x14, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x09, 0x2c, 0x00, 0x00, 0x00, 0x34, 0xfe, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xd7, 0xdf, 0xc3, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x22, 0xfe, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x09, 0x48, 0x00, 0x00, 0x00, 0x14, 0xfe, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x68, 0xa8, 0x04, 0x3e, 0x01, 0x00, 0x00, 0x00, 0xc0, 0x23, 0x04, 0x42, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x10, 0x00, 0x18, 0x00, 0x14, 0x00, 0x13, 0x00, + 0x00, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x07, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x48, 0x00, 0x00, 0x00, 0x8c, 0xfe, 0xff, 0xff, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x3b, 0xda, 0x75, 0x3b, 0x01, 0x00, 0x00, 0x00, 0x4f, 0xd8, 0xf5, 0x42, + 0x01, 0x00, 0x00, 0x00, 0xa8, 0x2a, 0x61, 0xc2, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x00, 0x6a, 0xff, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x2c, 0x00, 0x00, 0x00, 0x5c, 0xff, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xcf, 0x37, 0x69, 0x37, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0xb2, 0xff, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x2c, 0x00, 0x00, 0x00, 0xa4, 0xff, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0xd8, 0x72, 0x3b, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x18, 0x00, 0x14, 0x00, 0x13, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x04, 0x00, + 0x0e, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x3c, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x04, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xd4, 0x42, 0x16, 0x3c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x14, 0x00, 0x10, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x08, 0x00, 0x04, 0x00, + 0x0e, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x09, 0x54, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x14, 0x00, + 0x10, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x04, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa8, 0x41, 0x5b, 0x3d, 0x01, 0x00, 0x00, 0x00, 0x66, 0x66, 0x5a, 0x41, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x60, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, + 0xb4, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, + 0x8c, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, + 0x68, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, + 0x44, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, + 0x0c, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x04, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x96, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x72, 0x9e, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x19, + 0xa6, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x09, 0xae, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x1b, 0xb6, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x1b, + 0xbe, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x1b, 0xc6, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x09, 0xce, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x1b, + 0xd6, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x09, 0xde, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x1b, 0xe6, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x09, + 0xfa, 0xff, 0xff, 0xff, 0x00, 0x1b, 0x06, 0x00, 0x06, 0x00, 0x05, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x09, 0x06, 0x00, 0x08, 0x00, 0x07, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b}; + +const unsigned int g_keyword_scrambled_model_data_length = 34520; diff --git a/code/lib/tfmicro/tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h b/code/lib/tfmicro/tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h new file mode 100644 index 00000000..ce34426c --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h @@ -0,0 +1,22 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_BENCHMARKS_KEYWORD_SCRAMBLED_MODEL_DATA_H_ +#define TENSORFLOW_LITE_MICRO_BENCHMARKS_KEYWORD_SCRAMBLED_MODEL_DATA_H_ + +extern const unsigned char g_keyword_scrambled_model_data[]; +extern const unsigned int g_keyword_scrambled_model_data_length; + +#endif // TENSORFLOW_LITE_MICRO_BENCHMARKS_KEYWORD_SCRAMBLED_MODEL_DATA_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/debug_log.cc b/code/lib/tfmicro/tensorflow/lite/micro/debug_log.cc index 7ef582bd..46ca253a 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/debug_log.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/debug_log.cc @@ -1,4 +1,4 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -36,6 +36,15 @@ limitations under the License. #include "tensorflow/lite/micro/debug_log.h" +#ifndef TF_LITE_STRIP_ERROR_STRINGS #include +#endif -extern "C" void DebugLog(const char* s) { fprintf(stderr, "%s", s); } +extern "C" void DebugLog(const char* s) { +#ifndef TF_LITE_STRIP_ERROR_STRINGS + // Reusing TF_LITE_STRIP_ERROR_STRINGS to disable DebugLog completely to get + // maximum reduction in binary size. This is because we have DebugLog calls + // via TF_LITE_CHECK that are not stubbed out by TF_LITE_REPORT_ERROR. + fprintf(stderr, "%s", s); +#endif +} diff --git a/code/lib/tfmicro/tensorflow/lite/micro/debug_log.h b/code/lib/tfmicro/tensorflow/lite/micro/debug_log.h index 1004ab9f..c2840d0f 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/debug_log.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/debug_log.h @@ -15,9 +15,17 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_DEBUG_LOG_H_ #define TENSORFLOW_LITE_MICRO_DEBUG_LOG_H_ +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + // This function should be implemented by each target platform, and provide a // way for strings to be output to some text stream. For more information, see // tensorflow/lite/micro/debug_log.cc. -extern "C" void DebugLog(const char* s); +void DebugLog(const char* s); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus #endif // TENSORFLOW_LITE_MICRO_DEBUG_LOG_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/activation_utils.h b/code/lib/tfmicro/tensorflow/lite/micro/kernels/activation_utils.h index 7525bc93..95ecc26d 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/activation_utils.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/activation_utils.h @@ -21,6 +21,8 @@ limitations under the License. #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/kernels/internal/cppmath.h" +#include "tensorflow/lite/kernels/internal/max.h" +#include "tensorflow/lite/kernels/internal/min.h" namespace tflite { namespace ops { @@ -32,11 +34,11 @@ inline float ActivationValFloat(TfLiteFusedActivation act, float a) { case kTfLiteActNone: return a; case kTfLiteActRelu: - return std::max(0.0f, a); - case kTfLiteActRelu1: - return std::max(-1.0f, std::min(a, 1.0f)); + return TfLiteMax(0.0f, a); + case kTfLiteActReluN1To1: + return TfLiteMax(-1.0f, TfLiteMin(a, 1.0f)); case kTfLiteActRelu6: - return std::max(0.0f, std::min(a, 6.0f)); + return TfLiteMax(0.0f, TfLiteMin(a, 6.0f)); case kTfLiteActTanh: return std::tanh(a); case kTfLiteActSignBit: diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/activations.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/activations.cc index 4a9b8ce5..a92d5c73 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/activations.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/activations.cc @@ -18,30 +18,82 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/internal/types.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" #include "tensorflow/lite/micro/micro_utils.h" namespace tflite { namespace ops { namespace micro { namespace activations { +namespace { + +struct ReluOpData { + ReluParams params; +}; + +struct Relu6OpData { + int8_t six_int8; + int8_t zero_int8; + uint8_t six_uint8; + uint8_t zero_uint8; +}; + +} // namespace constexpr int kInputTensor = 0; constexpr int kOutputTensor = 0; -template -inline void ReluQuantized(int32_t lower, const RuntimeShape& input_shape, - const Q* input_data, const RuntimeShape& output_shape, - Q* output_data) { +template +inline void ReluQuantized(const ReluOpData& data, + const RuntimeShape& input_shape, + const RuntimeShape& output_shape, const T* input_data, + T* output_data) { const int flat_size = MatchingFlatSize(input_shape, output_shape); for (int i = 0; i < flat_size; ++i) { - const Q val = input_data[i]; - const Q clamped = val < lower ? lower : val; - output_data[i] = clamped; + const int32_t val = static_cast(input_data[i]); + int32_t clamped = + data.params.output_offset + + MultiplyByQuantizedMultiplier(val - data.params.input_offset, + data.params.output_multiplier, + data.params.output_shift); + clamped = std::max(data.params.quantized_activation_min, clamped); + clamped = std::min(data.params.quantized_activation_max, clamped); + output_data[i] = static_cast(clamped); } } +template +inline void CalculateReluOpData(const TfLiteTensor* input, TfLiteTensor* output, + ReluOpData* data) { + float act_min = 0.0; + float act_max = std::numeric_limits::infinity(); + double real_multiplier = + static_cast(input->params.scale / output->params.scale); + + const RuntimeShape input_shape = GetTensorShape(input); + const RuntimeShape output_shape = GetTensorShape(output); + + QuantizeMultiplier(real_multiplier, &data->params.output_multiplier, + &data->params.output_shift); + + data->params.quantized_activation_min = std::max( + static_cast(std::numeric_limits::min()), + output->params.zero_point + + static_cast(roundf(act_min / output->params.scale))); + data->params.quantized_activation_max = + act_max == std::numeric_limits::infinity() + ? static_cast(std::numeric_limits::max()) + : std::min(static_cast(std::numeric_limits::max()), + output->params.zero_point + + static_cast( + roundf(act_max / output->params.scale))); + data->params.input_offset = input->params.zero_point; + data->params.output_offset = output->params.zero_point; +} + inline void ReluFloat(const RuntimeShape& input_shape, const float* input_data, const RuntimeShape& output_shape, float* output_data) { const int flat_size = MatchingFlatSize(input_shape, output_shape); @@ -77,33 +129,59 @@ inline void Relu6Quantized(Q lower, Q upper, const RuntimeShape& input_shape, } } +void* ReluInit(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(ReluOpData)); +} + TfLiteStatus ReluPrepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + ReluOpData* data = static_cast(node->user_data); + + const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + if (input->type == kTfLiteInt8) { + CalculateReluOpData(input, output, data); + } else if (input->type == kTfLiteUInt8) { + CalculateReluOpData(input, output, data); + } + return kTfLiteOk; } TfLiteStatus ReluEval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TFLITE_DCHECK(node->user_data != nullptr); + const ReluOpData& data = *(static_cast(node->user_data)); + + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); switch (input->type) { case kTfLiteFloat32: { - ReluFloat(GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + ReluFloat(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } case kTfLiteInt8: { - ReluQuantized(input->params.zero_point, GetTensorShape(input), - GetTensorData(input), - GetTensorShape(output), - GetTensorData(output)); + ReluQuantized(data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } case kTfLiteUInt8: { - ReluQuantized(input->params.zero_point, GetTensorShape(input), - GetTensorData(input), - GetTensorShape(output), - GetTensorData(output)); + ReluQuantized(data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } default: { @@ -114,37 +192,63 @@ TfLiteStatus ReluEval(TfLiteContext* context, TfLiteNode* node) { } } +void* Relu6Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(Relu6OpData)); +} + TfLiteStatus Relu6Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + Relu6OpData* data = static_cast(node->user_data); + + const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); + + if (input->type == kTfLiteInt8) { + data->six_int8 = FloatToQuantizedType(6.0f, input->params.scale, + input->params.zero_point); + data->zero_int8 = input->params.zero_point; + } else if (input->type == kTfLiteUInt8) { + data->six_uint8 = FloatToQuantizedType(6.0f, input->params.scale, + input->params.zero_point); + data->zero_uint8 = input->params.zero_point; + } + return kTfLiteOk; } TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TFLITE_DCHECK(node->user_data != nullptr); + const Relu6OpData& data = *(static_cast(node->user_data)); + + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); switch (input->type) { case kTfLiteFloat32: { - Relu6Float(GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + Relu6Float(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } case kTfLiteInt8: { - const int8_t six = FloatToAsymmetricQuantizedInt8( - 6.0f, input->params.scale, input->params.zero_point); - const int8_t zero = input->params.zero_point; - Relu6Quantized( - zero, six, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + Relu6Quantized(data.zero_int8, data.six_int8, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } case kTfLiteUInt8: { - const uint8_t six = FloatToAsymmetricQuantizedUInt8( - 6.0f, input->params.scale, input->params.zero_point); - const uint8_t zero = input->params.zero_point; - Relu6Quantized( - zero, six, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + Relu6Quantized(data.zero_uint8, data.six_uint8, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } default: { @@ -157,28 +261,26 @@ TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace activations -TfLiteRegistration* Register_RELU() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/activations::ReluPrepare, - /*invoke=*/activations::ReluEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_RELU() { + return {/*init=*/activations::ReluInit, + /*free=*/nullptr, + /*prepare=*/activations::ReluPrepare, + /*invoke=*/activations::ReluEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_RELU6() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/activations::Relu6Prepare, - /*invoke=*/activations::Relu6Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_RELU6() { + return {/*init=*/activations::Relu6Init, + /*free=*/nullptr, + /*prepare=*/activations::Relu6Prepare, + /*invoke=*/activations::Relu6Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/add.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/add.cc index 42609301..e50d22cd 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/add.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/add.cc @@ -23,6 +23,8 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/memory_helpers.h" namespace tflite { namespace ops { @@ -40,18 +42,22 @@ struct OpData { // and the special 16-bit -> 16bit quantized path int input1_shift; int input2_shift; - int32 output_activation_min; - int32 output_activation_max; + int32_t output_activation_min; + int32_t output_activation_max; // These fields are used only in the general 8-bit -> 8bit quantized path - int32 input1_multiplier; - int32 input2_multiplier; - int32 output_multiplier; + int32_t input1_multiplier; + int32_t input2_multiplier; + int32_t output_multiplier; int output_shift; int left_shift; - int32 input1_offset; - int32 input2_offset; - int32 output_offset; + int32_t input1_offset; + int32_t input2_offset; + int32_t output_offset; + + // Used only for float evals: + float output_activation_min_f32; + float output_activation_max_f32; }; TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteAddParams* params, @@ -89,37 +95,44 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteAddParams* params, TF_LITE_ENSURE_STATUS(CalculateActivationRangeQuantized( context, params->activation, output, &data->output_activation_min, &data->output_activation_max)); + } else if (output->type == kTfLiteFloat32) { + CalculateActivationRange(params->activation, + &data->output_activation_min_f32, + &data->output_activation_max_f32); } return kTfLiteOk; } void EvalAdd(TfLiteContext* context, TfLiteNode* node, TfLiteAddParams* params, - const OpData* data, const TfLiteTensor* input1, - const TfLiteTensor* input2, TfLiteTensor* output) { - float output_activation_min, output_activation_max; - CalculateActivationRange(params->activation, &output_activation_min, - &output_activation_max); + const OpData* data, const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, TfLiteEvalTensor* output) { tflite::ArithmeticParams op_params; - SetActivationParams(output_activation_min, output_activation_max, &op_params); -#define TF_LITE_ADD(opname) \ - reference_ops::opname(op_params, GetTensorShape(input1), \ - GetTensorData(input1), GetTensorShape(input2), \ - GetTensorData(input2), GetTensorShape(output), \ - GetTensorData(output)) + SetActivationParams(data->output_activation_min_f32, + data->output_activation_max_f32, &op_params); if (data->requires_broadcast) { - TF_LITE_ADD(BroadcastAdd4DSlow); + reference_ops::BroadcastAdd4DSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_ADD(Add); + reference_ops::Add(op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } -#undef TF_LITE_ADD } TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, TfLiteAddParams* params, const OpData* data, - const TfLiteTensor* input1, - const TfLiteTensor* input2, - TfLiteTensor* output) { + const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, + TfLiteEvalTensor* output) { if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8) { tflite::ArithmeticParams op_params; op_params.left_shift = data->left_shift; @@ -135,46 +148,91 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, SetActivationParams(data->output_activation_min, data->output_activation_max, &op_params); bool need_broadcast = reference_ops::ProcessBroadcastShapes( - GetTensorShape(input1), GetTensorShape(input2), &op_params); -#define TF_LITE_ADD(type, opname, dtype) \ - type::opname(op_params, GetTensorShape(input1), \ - GetTensorData(input1), GetTensorShape(input2), \ - GetTensorData(input2), GetTensorShape(output), \ - GetTensorData(output)); + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorShape(input2), &op_params); if (output->type == kTfLiteInt8) { if (need_broadcast) { - TF_LITE_ADD(reference_integer_ops, BroadcastAdd4DSlow, int8_t); + reference_integer_ops::BroadcastAdd4DSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_ADD(reference_integer_ops, Add, int8_t); + reference_integer_ops::Add( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } else { if (need_broadcast) { - TF_LITE_ADD(reference_ops, BroadcastAdd4DSlow, uint8_t); + reference_ops::BroadcastAdd4DSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_ADD(reference_ops, Add, uint8_t); + reference_ops::Add(op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } -#undef TF_LITE_ADD } return kTfLiteOk; } +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); + + const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); + TF_LITE_ENSURE(context, input1 != nullptr); + const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); + TF_LITE_ENSURE(context, input2 != nullptr); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + OpData* data = static_cast(node->user_data); + auto* params = reinterpret_cast(node->builtin_data); + + TF_LITE_ENSURE_STATUS( + CalculateOpData(context, params, input1, input2, output, data)); + + return kTfLiteOk; +} + TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { auto* params = reinterpret_cast(node->builtin_data); - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); - OpData data; - TF_LITE_ENSURE_STATUS( - CalculateOpData(context, params, input1, input2, output, &data)); + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); if (output->type == kTfLiteFloat32) { - EvalAdd(context, node, params, &data, input1, input2, output); + EvalAdd(context, node, params, data, input1, input2, output); } else if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8) { - TF_LITE_ENSURE_OK(context, EvalAddQuantized(context, node, params, &data, + TF_LITE_ENSURE_OK(context, EvalAddQuantized(context, node, params, data, input1, input2, output)); } else { TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -187,16 +245,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace add -TfLiteRegistration* Register_ADD() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/add::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_ADD() { + return {/*init=*/add::Init, + /*free=*/nullptr, + /*prepare=*/add::Prepare, + /*invoke=*/add::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/all_ops_resolver.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/all_ops_resolver.cc deleted file mode 100644 index 6ba2e1aa..00000000 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/all_ops_resolver.cc +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/kernels/all_ops_resolver.h" - -#include "tensorflow/lite/micro/kernels/micro_ops.h" - -namespace tflite { -namespace ops { -namespace micro { - -// Register each supported op with: -// AddBuiltin(, , [min version], [max version]) -AllOpsResolver::AllOpsResolver() { - AddBuiltin(BuiltinOperator_FULLY_CONNECTED, Register_FULLY_CONNECTED(), 1, 4); - AddBuiltin(BuiltinOperator_MAX_POOL_2D, Register_MAX_POOL_2D(), 1, 2); - AddBuiltin(BuiltinOperator_SOFTMAX, Register_SOFTMAX(), 1, 2); - AddBuiltin(BuiltinOperator_LOGISTIC, Register_LOGISTIC(), 1, 2); - AddBuiltin(BuiltinOperator_SVDF, Register_SVDF(), 1, 3); - AddBuiltin(BuiltinOperator_CONV_2D, Register_CONV_2D(), 1, 3); - AddBuiltin(BuiltinOperator_CONCATENATION, Register_CONCATENATION(), 1, 3); - AddBuiltin(BuiltinOperator_DEPTHWISE_CONV_2D, Register_DEPTHWISE_CONV_2D(), 1, - 3); - AddBuiltin(BuiltinOperator_AVERAGE_POOL_2D, Register_AVERAGE_POOL_2D(), 1, 2); - AddBuiltin(BuiltinOperator_ABS, Register_ABS()); - AddBuiltin(BuiltinOperator_SIN, Register_SIN()); - AddBuiltin(BuiltinOperator_COS, Register_COS()); - AddBuiltin(BuiltinOperator_LOG, Register_LOG()); - AddBuiltin(BuiltinOperator_SQRT, Register_SQRT()); - AddBuiltin(BuiltinOperator_RSQRT, Register_RSQRT()); - AddBuiltin(BuiltinOperator_SQUARE, Register_SQUARE()); - AddBuiltin(BuiltinOperator_PRELU, Register_PRELU()); - AddBuiltin(BuiltinOperator_FLOOR, Register_FLOOR()); - AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM()); - AddBuiltin(BuiltinOperator_MINIMUM, Register_MINIMUM()); - AddBuiltin(BuiltinOperator_ARG_MAX, Register_ARG_MAX()); - AddBuiltin(BuiltinOperator_ARG_MIN, Register_ARG_MIN()); - AddBuiltin(BuiltinOperator_LOGICAL_OR, Register_LOGICAL_OR()); - AddBuiltin(BuiltinOperator_LOGICAL_AND, Register_LOGICAL_AND()); - AddBuiltin(BuiltinOperator_LOGICAL_NOT, Register_LOGICAL_NOT()); - AddBuiltin(BuiltinOperator_RESHAPE, Register_RESHAPE()); - AddBuiltin(BuiltinOperator_EQUAL, Register_EQUAL(), 1, 2); - AddBuiltin(BuiltinOperator_NOT_EQUAL, Register_NOT_EQUAL(), 1, 2); - AddBuiltin(BuiltinOperator_GREATER, Register_GREATER(), 1, 2); - AddBuiltin(BuiltinOperator_GREATER_EQUAL, Register_GREATER_EQUAL(), 1, 2); - AddBuiltin(BuiltinOperator_LESS, Register_LESS(), 1, 2); - AddBuiltin(BuiltinOperator_LESS_EQUAL, Register_LESS_EQUAL(), 1, 2); - AddBuiltin(BuiltinOperator_CEIL, Register_CEIL()); - AddBuiltin(BuiltinOperator_ROUND, Register_ROUND()); - AddBuiltin(BuiltinOperator_STRIDED_SLICE, Register_STRIDED_SLICE()); - AddBuiltin(BuiltinOperator_PACK, Register_PACK(), 1, 2); - AddBuiltin(BuiltinOperator_PAD, Register_PAD(), 1, 2); - AddBuiltin(BuiltinOperator_PADV2, Register_PADV2(), 1, 2); - AddBuiltin(BuiltinOperator_SPLIT, Register_SPLIT(), 1, 3); - AddBuiltin(BuiltinOperator_UNPACK, Register_UNPACK(), 1, 2); - AddBuiltin(BuiltinOperator_NEG, Register_NEG()); - AddBuiltin(BuiltinOperator_ADD, Register_ADD(), 1, 2); - AddBuiltin(BuiltinOperator_MUL, Register_MUL(), 1, 3); - AddBuiltin(BuiltinOperator_SUB, Register_SUB(), 1, 2); - AddBuiltin(BuiltinOperator_QUANTIZE, Register_QUANTIZE()); - AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE(), 1, 2); - AddBuiltin(BuiltinOperator_RELU, Register_RELU()); - AddBuiltin(BuiltinOperator_RELU6, Register_RELU6()); - AddBuiltin(BuiltinOperator_MEAN, Register_MEAN()); - AddBuiltin(BuiltinOperator_RESIZE_NEAREST_NEIGHBOR, - Register_RESIZE_NEAREST_NEIGHBOR(), - /* min_version = */ 1, - /* max_version = */ 2); - AddBuiltin(BuiltinOperator_L2_NORMALIZATION, Register_L2_NORMALIZATION()); -} - -} // namespace micro -} // namespace ops -} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/arg_min_max.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/arg_min_max.cc index a7c0a437..12ac0019 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/arg_min_max.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/arg_min_max.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" #include "tensorflow/lite/micro/kernels/micro_utils.h" namespace tflite { @@ -45,14 +46,20 @@ inline void ArgMinMaxHelper(const RuntimeShape& input1_shape, } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node, bool is_arg_max) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - const TfLiteTensor* axis = GetInput(context, node, kAxis); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + const TfLiteEvalTensor* axis = + tflite::micro::GetEvalInput(context, node, kAxis); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); -#define TF_LITE_ARG_MIN_MAX(data_type, axis_type, output_type) \ - ArgMinMaxHelper(GetTensorShape(input), GetTensorData(input), \ - GetTensorData(axis), GetTensorShape(output), \ - GetTensorData(output), is_arg_max) +#define TF_LITE_ARG_MIN_MAX(data_type, axis_type, output_type) \ + ArgMinMaxHelper(tflite::micro::GetTensorShape(input), \ + tflite::micro::GetTensorData(input), \ + tflite::micro::GetTensorData(axis), \ + tflite::micro::GetTensorShape(output), \ + tflite::micro::GetTensorData(output), \ + is_arg_max) if (axis->type == kTfLiteInt32) { if (output->type == kTfLiteInt32) { switch (input->type) { @@ -67,18 +74,19 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node, bool is_arg_max) { break; default: TF_LITE_KERNEL_LOG(context, - "Only float32, uint8 and int8 are " + "Only float32, uint8_t and int8_t are " "supported currently, got %s.", TfLiteTypeGetName(input->type)); return kTfLiteError; } } else { - TF_LITE_KERNEL_LOG(context, "Only int32 are supported currently, got %s.", + TF_LITE_KERNEL_LOG(context, + "Only int32_t are supported currently, got %s.", TfLiteTypeGetName(output->type)); return kTfLiteError; } } else { - TF_LITE_KERNEL_LOG(context, "Only int32 are supported currently, got %s.", + TF_LITE_KERNEL_LOG(context, "Only int32_t are supported currently, got %s.", TfLiteTypeGetName(axis->type)); return kTfLiteError; } @@ -98,28 +106,26 @@ TfLiteStatus ArgMaxEval(TfLiteContext* context, TfLiteNode* node) { } // namespace arg_min_max -TfLiteRegistration* Register_ARG_MAX() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/arg_min_max::ArgMaxEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_ARG_MAX() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/arg_min_max::ArgMaxEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_ARG_MIN() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/arg_min_max::ArgMinEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_ARG_MIN() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/arg_min_max::ArgMinEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/ceil.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/ceil.cc index 89831a76..f929ce62 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/ceil.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/ceil.cc @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -29,11 +30,13 @@ constexpr int kOutputTensor = 0; TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); - TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32); - TF_LITE_ENSURE_EQ(context, output->type, input->type); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, kTfLiteFloat32); + TF_LITE_ENSURE_TYPES_EQ(context, output->type, input->type); TF_LITE_ENSURE_EQ(context, output->bytes, input->bytes); TF_LITE_ENSURE_EQ(context, output->dims->size, input->dims->size); for (int i = 0; i < output->dims->size; ++i) { @@ -43,26 +46,29 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); - reference_ops::Ceil(GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + reference_ops::Ceil(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } } // namespace ceil -TfLiteRegistration* Register_CEIL() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ceil::Prepare, - /*invoke=*/ceil::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_CEIL() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ceil::Prepare, + /*invoke=*/ceil::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/circular_buffer.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/circular_buffer.cc index 590bdbe0..f7020306 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/circular_buffer.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/circular_buffer.cc @@ -17,11 +17,10 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" -#include "tensorflow/lite/kernels/internal/reference/integer_ops/add.h" -#include "tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" /* * The circular buffer custom operator is used to implement strided streaming @@ -78,7 +77,9 @@ void Free(TfLiteContext* context, void* buffer) { op_data_counter = 0; } TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); TF_LITE_ENSURE(context, input != nullptr); TF_LITE_ENSURE(context, output != nullptr); @@ -89,10 +90,10 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, 1, input->dims->data[2]); TF_LITE_ENSURE_EQ(context, output->dims->data[3], input->dims->data[3]); - TF_LITE_ENSURE_EQ(context, input->type, output->type); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); - // The circular buffer custom operator currently only supports int8. - TF_LITE_ENSURE_EQ(context, input->type, kTfLiteInt8); + // The circular buffer custom operator currently only supports int8_t. + TF_LITE_ENSURE_TYPES_EQ(context, input->type, kTfLiteInt8); // TODO(b/132070898): Use statically slotted OpData structures until a // scratch memory API is ready. @@ -121,8 +122,10 @@ void EvalInt8(const int8_t* input, int num_slots, int depth, int8_t* output) { } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); OpData* data = reinterpret_cast(node->user_data); @@ -130,8 +133,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { int depth = output->dims->data[3]; if (input->type == kTfLiteInt8) { - EvalInt8(GetTensorData(input), num_slots, depth, - GetTensorData(output)); + EvalInt8(tflite::micro::GetTensorData(input), num_slots, depth, + tflite::micro::GetTensorData(output)); } else { TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", TfLiteTypeGetName(input->type), input->type); diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/comparisons.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/comparisons.cc index 7db7af40..35007640 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/comparisons.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/comparisons.cc @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/quantization_util.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -25,103 +26,109 @@ namespace micro { namespace comparisons { namespace { +struct OpData { + ComparisonParams params; +}; + constexpr int kInputTensor1 = 0; constexpr int kInputTensor2 = 1; constexpr int kOutputTensor = 0; -// TODO(ruic): optimize macros below to using template functions. -#define TF_LITE_QUANTIZE_COMPARISON(opname) \ - template \ - void EvalQuantized##opname(TfLiteContext* context, TfLiteNode* node, \ - const TfLiteTensor* input1, \ - const TfLiteTensor* input2, TfLiteTensor* output, \ - bool requires_broadcast) { \ - if (input1->type == kTfLiteUInt8 || input1->type == kTfLiteInt8) { \ - auto input1_offset = -input1->params.zero_point; \ - auto input2_offset = -input2->params.zero_point; \ - const int left_shift = 8; \ - \ - int32 input1_multiplier; \ - int input1_shift; \ - QuantizeMultiplierSmallerThanOneExp( \ - static_cast(input1->params.scale), &input1_multiplier, \ - &input1_shift); \ - int32 input2_multiplier; \ - int input2_shift; \ - QuantizeMultiplierSmallerThanOneExp( \ - static_cast(input2->params.scale), &input2_multiplier, \ - &input2_shift); \ - \ - ComparisonParams op_params; \ - op_params.left_shift = left_shift; \ - op_params.input1_offset = input1_offset; \ - op_params.input1_multiplier = input1_multiplier; \ - op_params.input1_shift = input1_shift; \ - op_params.input2_offset = input2_offset; \ - op_params.input2_multiplier = input2_multiplier; \ - op_params.input2_shift = input2_shift; \ - if (requires_broadcast) { \ - reference_ops::Broadcast4DSlow##opname##WithScaling( \ - op_params, GetTensorShape(input1), \ - GetTensorData(input1), GetTensorShape(input2), \ - GetTensorData(input2), GetTensorShape(output), \ - GetTensorData(output)); \ - } else { \ - reference_ops::opname##WithScaling( \ - op_params, GetTensorShape(input1), \ - GetTensorData(input1), GetTensorShape(input2), \ - GetTensorData(input2), GetTensorShape(output), \ - GetTensorData(output)); \ - } \ - } \ - } -TF_LITE_QUANTIZE_COMPARISON(Equal); -TF_LITE_QUANTIZE_COMPARISON(NotEqual); -TF_LITE_QUANTIZE_COMPARISON(Greater); -TF_LITE_QUANTIZE_COMPARISON(GreaterEqual); -TF_LITE_QUANTIZE_COMPARISON(Less); -TF_LITE_QUANTIZE_COMPARISON(LessEqual); -#undef TF_LITE_QUANTIZE_COMPARISON - -#define TF_LITE_COMPARISON(type, opname, requires_broadcast) \ - { \ - ComparisonParams op_params; \ - requires_broadcast \ - ? reference_ops::Broadcast4DSlow##opname##NoScaling( \ - op_params, GetTensorShape(input1), GetTensorData(input1), \ - GetTensorShape(input2), GetTensorData(input2), \ - GetTensorShape(output), GetTensorData(output)) \ - : reference_ops::opname##NoScaling( \ - op_params, GetTensorShape(input1), GetTensorData(input1), \ - GetTensorShape(input2), GetTensorData(input2), \ - GetTensorShape(output), GetTensorData(output)); \ - } - TfLiteStatus EqualEval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - bool requires_broadcast = !HaveSameShapes(input1, input2); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); + + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + RuntimeShape input1_shape = tflite::micro::GetTensorShape(input1); + RuntimeShape input2_shape = tflite::micro::GetTensorShape(input2); + RuntimeShape output_shape = tflite::micro::GetTensorShape(output); + bool* output_data = tflite::micro::GetTensorData(output); + + bool requires_broadcast = !tflite::micro::HaveSameShapes(input1, input2); switch (input1->type) { case kTfLiteBool: - TF_LITE_COMPARISON(bool, Equal, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::EqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteFloat32: - TF_LITE_COMPARISON(float, Equal, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::EqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt32: - TF_LITE_COMPARISON(int32_t, Equal, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::EqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt64: - TF_LITE_COMPARISON(int64_t, Equal, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::EqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteUInt8: - EvalQuantizedEqual(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::EqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt8: - EvalQuantizedEqual(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::EqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -133,30 +140,100 @@ TfLiteStatus EqualEval(TfLiteContext* context, TfLiteNode* node) { // TODO(renjieliu): Refactor the logic to avoid duplications. TfLiteStatus NotEqualEval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - bool requires_broadcast = !HaveSameShapes(input1, input2); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); + + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + RuntimeShape input1_shape = tflite::micro::GetTensorShape(input1); + RuntimeShape input2_shape = tflite::micro::GetTensorShape(input2); + RuntimeShape output_shape = tflite::micro::GetTensorShape(output); + bool* output_data = tflite::micro::GetTensorData(output); + + bool requires_broadcast = !tflite::micro::HaveSameShapes(input1, input2); switch (input1->type) { case kTfLiteBool: - TF_LITE_COMPARISON(bool, NotEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowNotEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::NotEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteFloat32: - TF_LITE_COMPARISON(float, NotEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowNotEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::NotEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt32: - TF_LITE_COMPARISON(int32_t, NotEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowNotEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::NotEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt64: - TF_LITE_COMPARISON(int64_t, NotEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowNotEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::NotEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteUInt8: - EvalQuantizedNotEqual(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowNotEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::NotEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt8: - EvalQuantizedNotEqual(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowNotEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::NotEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -167,27 +244,87 @@ TfLiteStatus NotEqualEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus GreaterEval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - bool requires_broadcast = !HaveSameShapes(input1, input2); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); + + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + RuntimeShape input1_shape = tflite::micro::GetTensorShape(input1); + RuntimeShape input2_shape = tflite::micro::GetTensorShape(input2); + RuntimeShape output_shape = tflite::micro::GetTensorShape(output); + bool* output_data = tflite::micro::GetTensorData(output); + + bool requires_broadcast = !tflite::micro::HaveSameShapes(input1, input2); switch (input1->type) { case kTfLiteFloat32: - TF_LITE_COMPARISON(float, Greater, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt32: - TF_LITE_COMPARISON(int32_t, Greater, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt64: - TF_LITE_COMPARISON(int64_t, Greater, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteUInt8: - EvalQuantizedGreater(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt8: - EvalQuantizedGreater(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -198,27 +335,87 @@ TfLiteStatus GreaterEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus GreaterEqualEval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - bool requires_broadcast = !HaveSameShapes(input1, input2); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); + + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + RuntimeShape input1_shape = tflite::micro::GetTensorShape(input1); + RuntimeShape input2_shape = tflite::micro::GetTensorShape(input2); + RuntimeShape output_shape = tflite::micro::GetTensorShape(output); + bool* output_data = tflite::micro::GetTensorData(output); + + bool requires_broadcast = !tflite::micro::HaveSameShapes(input1, input2); switch (input1->type) { case kTfLiteFloat32: - TF_LITE_COMPARISON(float, GreaterEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt32: - TF_LITE_COMPARISON(int32_t, GreaterEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt64: - TF_LITE_COMPARISON(int64_t, GreaterEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteUInt8: - EvalQuantizedGreaterEqual(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt8: - EvalQuantizedGreaterEqual(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowGreaterEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::GreaterEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -229,27 +426,87 @@ TfLiteStatus GreaterEqualEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus LessEval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - bool requires_broadcast = !HaveSameShapes(input1, input2); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); + + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + RuntimeShape input1_shape = tflite::micro::GetTensorShape(input1); + RuntimeShape input2_shape = tflite::micro::GetTensorShape(input2); + RuntimeShape output_shape = tflite::micro::GetTensorShape(output); + bool* output_data = tflite::micro::GetTensorData(output); + + bool requires_broadcast = !tflite::micro::HaveSameShapes(input1, input2); switch (input1->type) { case kTfLiteFloat32: - TF_LITE_COMPARISON(float, Less, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt32: - TF_LITE_COMPARISON(int32_t, Less, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt64: - TF_LITE_COMPARISON(int64_t, Less, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteUInt8: - EvalQuantizedLess(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt8: - EvalQuantizedLess(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -260,27 +517,87 @@ TfLiteStatus LessEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus LessEqualEval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - bool requires_broadcast = !HaveSameShapes(input1, input2); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); + + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + RuntimeShape input1_shape = tflite::micro::GetTensorShape(input1); + RuntimeShape input2_shape = tflite::micro::GetTensorShape(input2); + RuntimeShape output_shape = tflite::micro::GetTensorShape(output); + bool* output_data = tflite::micro::GetTensorData(output); + + bool requires_broadcast = !tflite::micro::HaveSameShapes(input1, input2); switch (input1->type) { case kTfLiteFloat32: - TF_LITE_COMPARISON(float, LessEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt32: - TF_LITE_COMPARISON(int32_t, LessEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt64: - TF_LITE_COMPARISON(int64_t, LessEqual, requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessEqualNoScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteUInt8: - EvalQuantizedLessEqual(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; case kTfLiteInt8: - EvalQuantizedLessEqual(context, node, input1, input2, output, - requires_broadcast); + requires_broadcast + ? reference_ops::Broadcast4DSlowLessEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data) + : reference_ops::LessEqualWithScaling( + data->params, input1_shape, + tflite::micro::GetTensorData(input1), input2_shape, + tflite::micro::GetTensorData(input2), output_shape, + output_data); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -291,78 +608,115 @@ TfLiteStatus LessEqualEval(TfLiteContext* context, TfLiteNode* node) { } } // namespace + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + + const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); + TF_LITE_ENSURE(context, input1 != nullptr); + const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); + TF_LITE_ENSURE(context, input2 != nullptr); + + if (input1->type == kTfLiteUInt8 || input1->type == kTfLiteInt8) { + auto input1_offset = -input1->params.zero_point; + auto input2_offset = -input2->params.zero_point; + const int kLeftShift = 8; + + int32_t input1_multiplier; + int input1_shift; + QuantizeMultiplierSmallerThanOneExp( + static_cast(input1->params.scale), &input1_multiplier, + &input1_shift); + int32_t input2_multiplier; + int input2_shift; + QuantizeMultiplierSmallerThanOneExp( + static_cast(input2->params.scale), &input2_multiplier, + &input2_shift); + + data->params.left_shift = kLeftShift; + data->params.input1_offset = input1_offset; + data->params.input1_multiplier = input1_multiplier; + data->params.input1_shift = input1_shift; + data->params.input2_offset = input2_offset; + data->params.input2_multiplier = input2_multiplier; + data->params.input2_shift = input2_shift; + } + + return kTfLiteOk; +} + } // namespace comparisons -TfLiteRegistration* Register_EQUAL() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/comparisons::EqualEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_EQUAL() { + return {/*init=*/comparisons::Init, + /*free=*/nullptr, + /*prepare=*/comparisons::Prepare, + /*invoke=*/comparisons::EqualEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_NOT_EQUAL() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/comparisons::NotEqualEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_NOT_EQUAL() { + return {/*init=*/comparisons::Init, + /*free=*/nullptr, + /*prepare=*/comparisons::Prepare, + /*invoke=*/comparisons::NotEqualEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_GREATER() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/comparisons::GreaterEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_GREATER() { + return {/*init=*/comparisons::Init, + /*free=*/nullptr, + /*prepare=*/comparisons::Prepare, + /*invoke=*/comparisons::GreaterEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_GREATER_EQUAL() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/comparisons::GreaterEqualEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_GREATER_EQUAL() { + return {/*init=*/comparisons::Init, + /*free=*/nullptr, + /*prepare=*/comparisons::Prepare, + /*invoke=*/comparisons::GreaterEqualEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_LESS() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/comparisons::LessEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_LESS() { + return {/*init=*/comparisons::Init, + /*free=*/nullptr, + /*prepare=*/comparisons::Prepare, + /*invoke=*/comparisons::LessEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_LESS_EQUAL() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/comparisons::LessEqualEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_LESS_EQUAL() { + return {/*init=*/comparisons::Init, + /*free=*/nullptr, + /*prepare=*/comparisons::Prepare, + /*invoke=*/comparisons::LessEqualEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/concatenation.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/concatenation.cc index add4b126..8127cc32 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/concatenation.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/concatenation.cc @@ -18,10 +18,11 @@ limitations under the License. #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/kernels/internal/tensor.h" +#include "tensorflow/lite/kernels/internal/portable_tensor.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/internal/types.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -31,14 +32,116 @@ namespace concatenation { constexpr int kMaxInputNum = 10; // Maximum number of input tensors constexpr int kOutputTensor = 0; +struct OpData { + ConcatenationParams params; +}; + +// Handles negative axis index, coerces to positive index value. +inline int CalculatePositiveAxis(int axis, const TfLiteTensor* output_tensor) { + if (axis >= 0) { + return axis; + } else { + return NumDimensions(output_tensor) + axis; + } +} + +// The following functions are helpers to get tensor data in the format that the +// reference op implementation expects. They provide the same functionality as +// class VectorOfTensors and class VectorOfQuantizedTensors in TFLite. + +// Gets shapes from a list of tensors. +inline void GetAllInputTensorShapes(const TfLiteContext* context, + const TfLiteNode* node, + RuntimeShape all_shapes[kMaxInputNum]) { + TFLITE_DCHECK(context != nullptr); + TFLITE_DCHECK(node != nullptr); + for (int i = 0; i < node->inputs->size; ++i) { + const TfLiteEvalTensor* t = tflite::micro::GetEvalInput(context, node, i); + RuntimeShape shape = tflite::micro::GetTensorShape(t); + all_shapes[i].ReplaceWith(shape.DimensionsCount(), shape.DimsData()); + } +} + +// Get shape pointers from a list of shapes. +inline void GetShapesPointers(const RuntimeShape* shapes, size_t num, + const RuntimeShape* pointers[]) { + for (size_t i = 0; i < num; ++i) { + pointers[i] = &shapes[i]; + } +} + +// Gets data pointers from a list of tensors. +template +inline void GetAllInputTensorData(const TfLiteContext* context, + const TfLiteNode* node, + T* all_data[kMaxInputNum]) { + TFLITE_DCHECK(context != nullptr); + TFLITE_DCHECK(node != nullptr); + for (int i = 0; i < node->inputs->size; ++i) { + const TfLiteEvalTensor* t = tflite::micro::GetEvalInput(context, node, i); + all_data[i] = tflite::micro::GetTensorData(t); + } +} + +template +void EvalUnquantized(TfLiteContext* context, TfLiteNode* node) { + // Collect the shapes and data pointer of input tensors + RuntimeShape inputs_shape[kMaxInputNum]; + const RuntimeShape* inputs_shape_ptr[kMaxInputNum]; + const data_type* inputs_data[kMaxInputNum]; + GetAllInputTensorShapes(context, node, inputs_shape); + GetShapesPointers(inputs_shape, node->inputs->size, inputs_shape_ptr); + GetAllInputTensorData(context, node, inputs_data); + + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); + + reference_ops::Concatenation(data->params, inputs_shape_ptr, inputs_data, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +} + +void EvalQuantizedUInt8(TfLiteContext* context, TfLiteNode* node) { + // Collect the shapes and data pointer of input tensors + RuntimeShape inputs_shape[kMaxInputNum]; + const RuntimeShape* inputs_shape_ptr[kMaxInputNum]; + const uint8_t* inputs_data[kMaxInputNum]; + GetAllInputTensorShapes(context, node, inputs_shape); + GetShapesPointers(inputs_shape, node->inputs->size, inputs_shape_ptr); + GetAllInputTensorData(context, node, inputs_data); + + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); + + reference_ops::ConcatenationWithScaling( + data->params, inputs_shape_ptr, inputs_data, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +} + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { // This function only checks the types. Additional shape validations are // performed in the reference implementation called during Eval(). const TfLiteConcatenationParams* params = reinterpret_cast(node->builtin_data); - TfLiteType input_type = GetInput(context, node, 0)->type; - TfLiteType output_type = GetOutput(context, node, kOutputTensor)->type; + const TfLiteTensor* input_tensor = GetInput(context, node, 0); + TF_LITE_ENSURE(context, input_tensor != nullptr); + TfLiteType input_type = input_tensor->type; + const TfLiteTensor* output_tensor = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output_tensor != nullptr); + TfLiteType output_type = output_tensor->type; // Check activation and input type TF_LITE_ENSURE_EQ(context, params->activation, kTfLiteActNone); @@ -57,133 +160,76 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { // Shapes with dimensions >4 are not yet supported with static allocation. for (int i = 0; i < num_inputs; ++i) { const TfLiteTensor* input = GetInput(context, node, i); + TF_LITE_ENSURE(context, input != nullptr); int num_dimensions = NumDimensions(input); if (num_dimensions > 4) { TF_LITE_KERNEL_LOG( context, "Op Concatenation does not currently support num dimensions >4 " - "Tensor '%s' has %d dimensions.", - input->name, num_dimensions); + "Tensor has %d dimensions.", + num_dimensions); return kTfLiteError; } } + // Calculate OpData. + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + switch (output_type) { // Already know in/outtypes are same. + case kTfLiteFloat32: + case kTfLiteInt32: + case kTfLiteInt64: { + data->params.axis = CalculatePositiveAxis(params->axis, output); + data->params.inputs_count = node->inputs->size; + break; + } + case kTfLiteUInt8: + case kTfLiteInt8: { + data->params.axis = CalculatePositiveAxis(params->axis, output); + data->params.inputs_count = node->inputs->size; + + float* input_scales = + reinterpret_cast(context->AllocatePersistentBuffer( + context, node->inputs->size * sizeof(float))); + + int32_t* input_zero_points = + reinterpret_cast(context->AllocatePersistentBuffer( + context, node->inputs->size * sizeof(int32_t))); + + // Allocate persistent scale and zeropoint buffers. + // Store input scale and zero point values in OpParams: + for (int i = 0; i < node->inputs->size; ++i) { + const TfLiteTensor* t = GetInput(context, node, i); + TF_LITE_ENSURE(context, t != nullptr); + input_scales[i] = t->params.scale; + input_zero_points[i] = t->params.zero_point; + } + + data->params.input_scale = input_scales; + data->params.input_zeropoint = input_zero_points; + data->params.output_zeropoint = output->params.zero_point; + data->params.output_scale = output->params.scale; + break; + } + default: + TF_LITE_KERNEL_LOG( + context, "Op Concatenation does not currently support Type '%s'.", + TfLiteTypeGetName(output_type)); + return kTfLiteError; + } + return kTfLiteOk; } -// Handles negative axis index, coerces to positive index value. -inline int CalculatePositiveAxis(int axis, const TfLiteTensor* output_tensor) { - if (axis >= 0) { - return axis; - } else { - return NumDimensions(output_tensor) + axis; - } -} - -// The following functions are helpers to get tensor data in the format that the -// reference op implementation expects. They provide the same functionality as -// class VectorOfTensors and class VectorOfQuantizedTensors in TFLite. - -// Gets shapes from a list of tensors. -inline void GetAllTensorShapes(const TfLiteContext& context, - const TfLiteIntArray& tensor_list, - RuntimeShape all_shapes[kMaxInputNum]) { - for (int i = 0; i < tensor_list.size; ++i) { - const TfLiteTensor* t = &context.tensors[tensor_list.data[i]]; - RuntimeShape shape = GetTensorShape(t); - all_shapes[i].ReplaceWith(shape.DimensionsCount(), shape.DimsData()); - } -} - -// Get shape pointers from a list of shapes. -inline void GetShapesPointers(const RuntimeShape* shapes, size_t num, - const RuntimeShape* pointers[]) { - for (size_t i = 0; i < num; ++i) { - pointers[i] = &shapes[i]; - } -} - -// Gets data pointers from a list of tensors. -template -inline void GetAllTensorData(const TfLiteContext& context, - const TfLiteIntArray& tensor_list, - T* all_data[kMaxInputNum]) { - for (int i = 0; i < tensor_list.size; ++i) { - const TfLiteTensor* t = &context.tensors[tensor_list.data[i]]; - all_data[i] = GetTensorData(t); - } -} - -// Gets scale and zero point from a list of tensors -inline void GetAllQuantizationParam(const TfLiteContext& context, - const TfLiteIntArray& tensor_list, - float scales[kMaxInputNum], - int32 zero_points[kMaxInputNum]) { - for (int i = 0; i < tensor_list.size; ++i) { - const TfLiteTensor* t = &context.tensors[tensor_list.data[i]]; - scales[i] = t->params.scale; - zero_points[i] = t->params.zero_point; - } -} - -template -void EvalUnquantized(TfLiteContext* context, TfLiteNode* node) { - // Collect the shapes and data pointer of input tensors - RuntimeShape inputs_shape[kMaxInputNum]; - const RuntimeShape* inputs_shape_ptr[kMaxInputNum]; - const data_type* inputs_data[kMaxInputNum]; - GetAllTensorShapes(*context, *node->inputs, inputs_shape); - GetShapesPointers(inputs_shape, node->inputs->size, inputs_shape_ptr); - GetAllTensorData(*context, *node->inputs, inputs_data); - - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - - const TfLiteConcatenationParams* params = - reinterpret_cast(node->builtin_data); - - ConcatenationParams op_params; - op_params.axis = CalculatePositiveAxis(params->axis, output); - op_params.inputs_count = NumInputs(node); - - reference_ops::Concatenation(op_params, inputs_shape_ptr, inputs_data, - GetTensorShape(output), - GetTensorData(output)); -} - -void EvalQuantizedUInt8(TfLiteContext* context, TfLiteNode* node) { - // Collect the shapes and data pointer of input tensors - RuntimeShape inputs_shape[kMaxInputNum]; - const RuntimeShape* inputs_shape_ptr[kMaxInputNum]; - const uint8_t* inputs_data[kMaxInputNum]; - float inputs_scale[kMaxInputNum]; - int32 inputs_zero_point[kMaxInputNum]; - GetAllTensorShapes(*context, *node->inputs, inputs_shape); - GetShapesPointers(inputs_shape, node->inputs->size, inputs_shape_ptr); - GetAllTensorData(*context, *node->inputs, inputs_data); - GetAllQuantizationParam(*context, *node->inputs, inputs_scale, - inputs_zero_point); - - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - - const TfLiteConcatenationParams* params = - reinterpret_cast(node->builtin_data); - - ConcatenationParams op_params; - op_params.axis = CalculatePositiveAxis(params->axis, output); - op_params.inputs_count = NumInputs(node); - op_params.input_zeropoint = inputs_zero_point; - op_params.input_scale = inputs_scale; - op_params.output_zeropoint = output->params.zero_point; - op_params.output_scale = output->params.scale; - - reference_ops::ConcatenationWithScaling(op_params, inputs_shape_ptr, - inputs_data, GetTensorShape(output), - GetTensorData(output)); -} - TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - TfLiteType output_type = GetOutput(context, node, kOutputTensor)->type; + const TfLiteTensor* output_tensor = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output_tensor != nullptr); + TfLiteType output_type = output_tensor->type; switch (output_type) { // Already know in/outtypes are same. case kTfLiteFloat32: @@ -214,16 +260,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace concatenation -TfLiteRegistration* Register_CONCATENATION() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/concatenation::Prepare, - /*invoke=*/concatenation::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_CONCATENATION() { + return {/*init=*/concatenation::Init, + /*free=*/nullptr, + /*prepare=*/concatenation::Prepare, + /*invoke=*/concatenation::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/conv copy.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/conv copy.cc deleted file mode 100644 index a82fd53c..00000000 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/conv copy.cc +++ /dev/null @@ -1,279 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/kernels/internal/reference/conv.h" - -#include "tensorflow/lite/c/builtin_op_data.h" -#include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/kernels/internal/common.h" -#include "tensorflow/lite/kernels/internal/quantization_util.h" -#include "tensorflow/lite/kernels/internal/reference/integer_ops/conv.h" -#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" -#include "tensorflow/lite/kernels/kernel_util.h" -#include "tensorflow/lite/kernels/padding.h" - -namespace tflite { -namespace ops { -namespace micro { -namespace conv { - -constexpr int kInputTensor = 0; -constexpr int kFilterTensor = 1; -constexpr int kBiasTensor = 2; -constexpr int kOutputTensor = 0; -// Angepasst jomjol 05.06.20 -//constexpr int kMaxChannels = 1024; -constexpr int kMaxChannels = 4096; - -// Conv is quantized along dimension 0: -// https://www.tensorflow.org/lite/performance/quantization_spec -constexpr int kConvQuantizedDimension = 0; - -// This file has 2 implementation of Conv. - -struct OpData { - TfLitePaddingValues padding; - // The scaling factor from input to output (aka the 'real multiplier') can - // be represented as a fixed point multiplier plus a left shift. - int32_t output_multiplier; - int output_shift; - - // Per channel output multiplier and shift. - // TODO(b/141139247): Allocate these dynamically when possible. - int32_t per_channel_output_multiplier[kMaxChannels]; - int32_t per_channel_output_shift[kMaxChannels]; - - // The range of the fused activation layer. For example for kNone and - // uint8_t these would be 0 and 255. - int32_t output_activation_min; - int32_t output_activation_max; -}; - -inline PaddingType RuntimePaddingType(TfLitePadding padding) { - switch (padding) { - case TfLitePadding::kTfLitePaddingSame: - return PaddingType::kSame; - case TfLitePadding::kTfLitePaddingValid: - return PaddingType::kValid; - case TfLitePadding::kTfLitePaddingUnknown: - default: - return PaddingType::kNone; - } -} - -TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, int width, int height, - int filter_width, int filter_height, int out_width, - int out_height, const TfLiteType data_type, - OpData* data) { - bool has_bias = node->inputs->size == 3; - // Check number of inputs/outputs - TF_LITE_ENSURE(context, has_bias || node->inputs->size == 2); - TF_LITE_ENSURE_EQ(context, node->outputs->size, 1); - - // Matching GetWindowedOutputSize in TensorFlow. - auto padding = params->padding; - data->padding = ComputePaddingHeightWidth( - params->stride_height, params->stride_width, - params->dilation_height_factor, params->dilation_width_factor, height, - width, filter_height, filter_width, padding, &out_height, &out_width); - - // Note that quantized inference requires that all tensors have their - // parameters set. This is usually done during quantized training. - if (data_type != kTfLiteFloat32) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); - const TfLiteTensor* bias = - GetOptionalInputTensor(context, node, kBiasTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - int output_channels = filter->dims->data[kConvQuantizedDimension]; - - TF_LITE_ENSURE_STATUS(tflite::PopulateConvolutionQuantizationParams( - context, input, filter, bias, output, params->activation, - &data->output_multiplier, &data->output_shift, - &data->output_activation_min, &data->output_activation_max, - data->per_channel_output_multiplier, - reinterpret_cast(data->per_channel_output_shift), - output_channels)); - } - return kTfLiteOk; -} - -void EvalQuantized(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* im2col, - TfLiteTensor* hwcn_weights, TfLiteTensor* output) { - const int32_t input_offset = -input->params.zero_point; - const int32_t filter_offset = -filter->params.zero_point; - const int32_t output_offset = output->params.zero_point; - - ConvParams op_params; - op_params.padding_type = RuntimePaddingType(params->padding); - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; - op_params.stride_width = params->stride_width; - op_params.stride_height = params->stride_height; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.input_offset = input_offset; - op_params.weights_offset = filter_offset; - op_params.output_offset = output_offset; - op_params.output_multiplier = data->output_multiplier; - op_params.output_shift = -data->output_shift; - op_params.quantized_activation_min = data->output_activation_min; - op_params.quantized_activation_max = data->output_activation_max; - reference_ops::Conv(op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output), GetTensorShape(im2col), - GetTensorData(im2col), nullptr); -} - -void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - const TfLiteTensor* input, - const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* output, - TfLiteTensor* im2col) { - ConvParams op_params; - op_params.input_offset = -input->params.zero_point; - op_params.output_offset = output->params.zero_point; - op_params.stride_height = params->stride_height; - op_params.stride_width = params->stride_width; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.padding_values.height = data->padding.height; - op_params.padding_values.width = data->padding.width; - op_params.quantized_activation_min = data->output_activation_min; - op_params.quantized_activation_max = data->output_activation_max; - - reference_integer_ops::ConvPerChannel( - op_params, data->per_channel_output_multiplier, - data->per_channel_output_shift, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output)); -} - -void EvalFloat(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* im2col, - TfLiteTensor* hwcn_weights, TfLiteTensor* output) { - float output_activation_min, output_activation_max; - CalculateActivationRange(params->activation, &output_activation_min, - &output_activation_max); - - ConvParams op_params; - op_params.padding_type = RuntimePaddingType(params->padding); - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; - op_params.stride_width = params->stride_width; - op_params.stride_height = params->stride_height; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.float_activation_min = output_activation_min; - op_params.float_activation_max = output_activation_max; - - reference_ops::Conv(op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output), GetTensorShape(im2col), - GetTensorData(im2col)); -} - -TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - auto* params = reinterpret_cast(node->builtin_data); - - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); - const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); - - int input_width = input->dims->data[2]; - int input_height = input->dims->data[1]; - int filter_width = filter->dims->data[2]; - int filter_height = filter->dims->data[1]; - int output_width = output->dims->data[2]; - int output_height = output->dims->data[1]; - - OpData data; - - // All per-channel quantized tensors need valid zero point and scale arrays. - if (input->type == kTfLiteInt8) { - TF_LITE_ENSURE_EQ(context, filter->quantization.type, - kTfLiteAffineQuantization); - - const auto* affine_quantization = - reinterpret_cast( - filter->quantization.params); - TF_LITE_ENSURE(context, affine_quantization); - TF_LITE_ENSURE(context, affine_quantization->scale); - TF_LITE_ENSURE(context, affine_quantization->zero_point); - - TF_LITE_ENSURE(context, - affine_quantization->scale->size == 1 || - affine_quantization->scale->size == - filter->dims->data[kConvQuantizedDimension]); - TF_LITE_ENSURE_EQ(context, affine_quantization->scale->size, - affine_quantization->zero_point->size); - } - - TF_LITE_ENSURE_STATUS(CalculateOpData( - context, node, params, input_width, input_height, filter_width, - filter_height, output_width, output_height, input->type, &data)); - - switch (input->type) { // Already know in/out types are same. - case kTfLiteFloat32: - EvalFloat(context, node, params, &data, input, filter, bias, nullptr, - nullptr, output); - break; - case kTfLiteInt8: - EvalQuantizedPerChannel(context, node, params, &data, input, filter, bias, - output, nullptr); - break; - case kTfLiteUInt8: - EvalQuantized(context, node, params, &data, input, filter, bias, nullptr, - nullptr, output); - break; - default: - TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", - TfLiteTypeGetName(input->type), input->type); - return kTfLiteError; - } - return kTfLiteOk; -} - -} // namespace conv - -TfLiteRegistration* Register_CONV_2D() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/conv::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; -} - -} // namespace micro -} // namespace ops -} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/conv.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/conv.cc index 393791f0..55efa486 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/conv.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/conv.cc @@ -23,19 +23,15 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/padding.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { -namespace ops { -namespace micro { -namespace conv { +namespace { constexpr int kInputTensor = 0; constexpr int kFilterTensor = 1; constexpr int kBiasTensor = 2; constexpr int kOutputTensor = 0; -// Angepasst jomjol 05.06.20 -//constexpr int kMaxChannels = 1024; -constexpr int kMaxChannels = 32384; // Conv is quantized along dimension 0: // https://www.tensorflow.org/lite/performance/quantization_spec @@ -45,15 +41,20 @@ constexpr int kConvQuantizedDimension = 0; struct OpData { TfLitePaddingValues padding; + + // Cached tensor zero point values for quantized operations. + int32_t input_zero_point; + int32_t filter_zero_point; + int32_t output_zero_point; + // The scaling factor from input to output (aka the 'real multiplier') can // be represented as a fixed point multiplier plus a left shift. int32_t output_multiplier; int output_shift; // Per channel output multiplier and shift. - // TODO(b/141139247): Allocate these dynamically when possible. - int32_t per_channel_output_multiplier[kMaxChannels]; - int32_t per_channel_output_shift[kMaxChannels]; + int32_t* per_channel_output_multiplier; + int32_t* per_channel_output_shift; // The range of the fused activation layer. For example for kNone and // uint8_t these would be 0 and 255. @@ -74,10 +75,10 @@ inline PaddingType RuntimePaddingType(TfLitePadding padding) { } TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, int width, int height, - int filter_width, int filter_height, int out_width, - int out_height, const TfLiteType data_type, - OpData* data) { + const TfLiteConvParams* params, int width, + int height, int filter_width, int filter_height, + int out_width, int out_height, + const TfLiteType data_type, OpData* data) { bool has_bias = node->inputs->size == 3; // Check number of inputs/outputs TF_LITE_ENSURE(context, has_bias || node->inputs->size == 2); @@ -94,10 +95,13 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, // parameters set. This is usually done during quantized training. if (data_type != kTfLiteFloat32) { const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); + TF_LITE_ENSURE(context, filter != nullptr); const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); int output_channels = filter->dims->data[kConvQuantizedDimension]; TF_LITE_ENSURE_STATUS(tflite::PopulateConvolutionQuantizationParams( @@ -111,100 +115,24 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, return kTfLiteOk; } -void EvalQuantized(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* im2col, - TfLiteTensor* hwcn_weights, TfLiteTensor* output) { - const int32_t input_offset = -input->params.zero_point; - const int32_t filter_offset = -filter->params.zero_point; - const int32_t output_offset = output->params.zero_point; - - ConvParams op_params; - op_params.padding_type = RuntimePaddingType(params->padding); - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; - op_params.stride_width = params->stride_width; - op_params.stride_height = params->stride_height; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.input_offset = input_offset; - op_params.weights_offset = filter_offset; - op_params.output_offset = output_offset; - op_params.output_multiplier = data->output_multiplier; - op_params.output_shift = -data->output_shift; - op_params.quantized_activation_min = data->output_activation_min; - op_params.quantized_activation_max = data->output_activation_max; - reference_ops::Conv(op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output), GetTensorShape(im2col), - GetTensorData(im2col), nullptr); +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); } -void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - const TfLiteTensor* input, - const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* output, - TfLiteTensor* im2col) { - ConvParams op_params; - op_params.input_offset = -input->params.zero_point; - op_params.output_offset = output->params.zero_point; - op_params.stride_height = params->stride_height; - op_params.stride_width = params->stride_width; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.padding_values.height = data->padding.height; - op_params.padding_values.width = data->padding.width; - op_params.quantized_activation_min = data->output_activation_min; - op_params.quantized_activation_max = data->output_activation_max; +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); - reference_integer_ops::ConvPerChannel( - op_params, data->per_channel_output_multiplier, - data->per_channel_output_shift, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output)); -} - -void EvalFloat(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* im2col, - TfLiteTensor* hwcn_weights, TfLiteTensor* output) { - float output_activation_min, output_activation_max; - CalculateActivationRange(params->activation, &output_activation_min, - &output_activation_max); - - ConvParams op_params; - op_params.padding_type = RuntimePaddingType(params->padding); - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; - op_params.stride_width = params->stride_width; - op_params.stride_height = params->stride_height; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.float_activation_min = output_activation_min; - op_params.float_activation_max = output_activation_max; - - reference_ops::Conv(op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output), GetTensorShape(im2col), - GetTensorData(im2col)); -} - -TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - auto* params = reinterpret_cast(node->builtin_data); + OpData* data = static_cast(node->user_data); + const auto params = static_cast(node->builtin_data); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); - const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); + TF_LITE_ENSURE(context, filter != nullptr); int input_width = input->dims->data[2]; int input_height = input->dims->data[1]; @@ -212,9 +140,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { int filter_height = filter->dims->data[1]; int output_width = output->dims->data[2]; int output_height = output->dims->data[1]; - - struct tflite::ops::micro::conv::OpData *data = (struct tflite::ops::micro::conv::OpData*) malloc(sizeof(struct tflite::ops::micro::conv::OpData)); + // Dynimically allocate per-channel quantization parameters. + const int num_channels = filter->dims->data[kConvQuantizedDimension]; + data->per_channel_output_multiplier = + static_cast(context->AllocatePersistentBuffer( + context, num_channels * sizeof(int32_t))); + data->per_channel_output_shift = + static_cast(context->AllocatePersistentBuffer( + context, num_channels * sizeof(int32_t))); // All per-channel quantized tensors need valid zero point and scale arrays. if (input->type == kTfLiteInt8) { @@ -222,8 +156,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { kTfLiteAffineQuantization); const auto* affine_quantization = - reinterpret_cast( - filter->quantization.params); + static_cast(filter->quantization.params); TF_LITE_ENSURE(context, affine_quantization); TF_LITE_ENSURE(context, affine_quantization->scale); TF_LITE_ENSURE(context, affine_quantization->zero_point); @@ -240,6 +173,136 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { context, node, params, input_width, input_height, filter_width, filter_height, output_width, output_height, input->type, data)); + data->input_zero_point = input->params.zero_point; + data->filter_zero_point = filter->params.zero_point; + data->output_zero_point = output->params.zero_point; + + return kTfLiteOk; +} // namespace conv + +void EvalQuantized(TfLiteContext* context, TfLiteNode* node, + TfLiteConvParams* params, const OpData& data, + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* filter, const TfLiteEvalTensor* bias, + TfLiteEvalTensor* im2col, TfLiteEvalTensor* hwcn_weights, + TfLiteEvalTensor* output) { + const int32_t input_offset = -data.input_zero_point; + const int32_t filter_offset = -data.filter_zero_point; + const int32_t output_offset = data.output_zero_point; + + // TODO(b/154032858): Investigate removing extra copies. + ConvParams op_params; + op_params.padding_type = RuntimePaddingType(params->padding); + op_params.padding_values.width = data.padding.width; + op_params.padding_values.height = data.padding.height; + op_params.stride_width = params->stride_width; + op_params.stride_height = params->stride_height; + op_params.dilation_width_factor = params->dilation_width_factor; + op_params.dilation_height_factor = params->dilation_height_factor; + op_params.input_offset = input_offset; + op_params.weights_offset = filter_offset; + op_params.output_offset = output_offset; + op_params.output_multiplier = data.output_multiplier; + op_params.output_shift = -data.output_shift; + op_params.quantized_activation_min = data.output_activation_min; + op_params.quantized_activation_max = data.output_activation_max; + reference_ops::Conv(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), + tflite::micro::GetTensorShape(im2col), + tflite::micro::GetTensorData(im2col), nullptr); +} + +void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, + TfLiteConvParams* params, const OpData& data, + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* filter, + const TfLiteEvalTensor* bias, + TfLiteEvalTensor* output, + TfLiteEvalTensor* im2col) { + // TODO(b/154032858): Investigate removing extra copies. + ConvParams op_params; + op_params.input_offset = -data.input_zero_point; + op_params.output_offset = data.output_zero_point; + op_params.stride_height = params->stride_height; + op_params.stride_width = params->stride_width; + op_params.dilation_height_factor = params->dilation_height_factor; + op_params.dilation_width_factor = params->dilation_width_factor; + op_params.padding_values.height = data.padding.height; + op_params.padding_values.width = data.padding.width; + op_params.quantized_activation_min = data.output_activation_min; + op_params.quantized_activation_max = data.output_activation_max; + + reference_integer_ops::ConvPerChannel( + op_params, data.per_channel_output_multiplier, + data.per_channel_output_shift, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +} + +void EvalFloat(TfLiteContext* context, TfLiteNode* node, + TfLiteConvParams* params, const OpData& data, + const TfLiteEvalTensor* input, const TfLiteEvalTensor* filter, + const TfLiteEvalTensor* bias, TfLiteEvalTensor* im2col, + TfLiteEvalTensor* hwcn_weights, TfLiteEvalTensor* output) { + float output_activation_min, output_activation_max; + CalculateActivationRange(params->activation, &output_activation_min, + &output_activation_max); + // TODO(b/154032858): Investigate removing extra copies. + ConvParams op_params; + op_params.padding_type = RuntimePaddingType(params->padding); + op_params.padding_values.width = data.padding.width; + op_params.padding_values.height = data.padding.height; + op_params.stride_width = params->stride_width; + op_params.stride_height = params->stride_height; + op_params.dilation_width_factor = params->dilation_width_factor; + op_params.dilation_height_factor = params->dilation_height_factor; + op_params.float_activation_min = output_activation_min; + op_params.float_activation_max = output_activation_max; + + reference_ops::Conv(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), + tflite::micro::GetTensorShape(im2col), + tflite::micro::GetTensorData(im2col)); +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + auto* params = reinterpret_cast(node->builtin_data); + + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + const TfLiteEvalTensor* filter = + tflite::micro::GetEvalInput(context, node, kFilterTensor); + const TfLiteEvalTensor* bias = + (NumInputs(node) == 3) + ? tflite::micro::GetEvalInput(context, node, kBiasTensor) + : nullptr; + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + TFLITE_DCHECK(node->user_data != nullptr); + const OpData& data = *(static_cast(node->user_data)); + + TF_LITE_ENSURE_EQ(context, input->type, output->type); + TF_LITE_ENSURE_MSG(context, input->type == filter->type, + "Hybrid models are not supported on TFLite Micro."); + switch (input->type) { // Already know in/out types are same. case kTfLiteFloat32: EvalFloat(context, node, params, data, input, filter, bias, nullptr, @@ -256,27 +319,22 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", TfLiteTypeGetName(input->type), input->type); - free(data); return kTfLiteError; } - free(data); return kTfLiteOk; } -} // namespace conv +} // namespace -TfLiteRegistration* Register_CONV_2D() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/conv::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_CONV_2D() { + return {/*init=*/Init, + /*free=*/nullptr, + /*prepare=*/Prepare, + /*invoke=*/Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -} // namespace micro -} // namespace ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/depthwise_conv.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/depthwise_conv.cc index 8618646a..85b51233 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/depthwise_conv.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/depthwise_conv.cc @@ -24,18 +24,15 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/padding.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { -namespace ops { -namespace micro { -namespace depthwise_conv { namespace { constexpr int kInputTensor = 0; constexpr int kFilterTensor = 1; constexpr int kBiasTensor = 2; constexpr int kOutputTensor = 0; -constexpr int kMaxChannels = 1024; // Depthwise conv is quantized along dimension 3: // https://www.tensorflow.org/lite/performance/quantization_spec @@ -43,16 +40,20 @@ constexpr int kDepthwiseConvQuantizedDimension = 3; struct OpData { TfLitePaddingValues padding; + + // Cached tensor zero point values for quantized operations. + int32_t input_zero_point; + int32_t filter_zero_point; + int32_t output_zero_point; + // The scaling factor from input to output (aka the 'real multiplier') can // be represented as a fixed point multiplier plus a left shift. int32_t output_multiplier; int output_shift; // Per channel output multiplier and shift. - // TODO(b/141139247): Allocate these dynamically when possible. - int32_t per_channel_output_multiplier[kMaxChannels]; - int32_t per_channel_output_shift[kMaxChannels]; - + int32_t* per_channel_output_multiplier; + int32_t* per_channel_output_shift; // The range of the fused activation layer. For example for kNone and // uint8_t these would be 0 and 255. int32_t output_activation_min; @@ -78,125 +79,44 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, // parameters set. This is usually done during quantized training. if (data_type != kTfLiteFloat32) { const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); + TF_LITE_ENSURE(context, filter != nullptr); const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); int num_channels = filter->dims->data[kDepthwiseConvQuantizedDimension]; - TF_LITE_ENSURE_STATUS(tflite::PopulateConvolutionQuantizationParams( + return tflite::PopulateConvolutionQuantizationParams( context, input, filter, bias, output, params->activation, &data->output_multiplier, &data->output_shift, &data->output_activation_min, &data->output_activation_max, data->per_channel_output_multiplier, - reinterpret_cast(data->per_channel_output_shift), num_channels)); + reinterpret_cast(data->per_channel_output_shift), num_channels); } return kTfLiteOk; } -} // namespace - -void EvalFloat(TfLiteContext* context, TfLiteNode* node, - TfLiteDepthwiseConvParams* params, OpData* data, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* output) { - float output_activation_min, output_activation_max; - CalculateActivationRange(params->activation, &output_activation_min, - &output_activation_max); - - tflite::DepthwiseParams op_params; - // Padding type is ignored, but still set. - op_params.padding_type = PaddingType::kSame; - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; - op_params.stride_width = params->stride_width; - op_params.stride_height = params->stride_height; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.depth_multiplier = params->depth_multiplier; - op_params.float_activation_min = output_activation_min; - op_params.float_activation_max = output_activation_max; - - tflite::reference_ops::DepthwiseConv( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(filter), GetTensorData(filter), - GetTensorShape(bias), GetTensorData(bias), GetTensorShape(output), - GetTensorData(output)); +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); } -void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, - TfLiteDepthwiseConvParams* params, OpData* data, - const TfLiteTensor* input, - const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* output) { - DepthwiseParams op_params; - op_params.padding_type = PaddingType::kSame; - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; - op_params.stride_width = params->stride_width; - op_params.stride_height = params->stride_height; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.depth_multiplier = params->depth_multiplier; - op_params.input_offset = -input->params.zero_point; - op_params.weights_offset = 0; - op_params.output_offset = output->params.zero_point; - // TODO(b/130439627): Use calculated value for clamping. - op_params.quantized_activation_min = std::numeric_limits::min(); - op_params.quantized_activation_max = std::numeric_limits::max(); +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); - reference_integer_ops::DepthwiseConvPerChannel( - op_params, data->per_channel_output_multiplier, - data->per_channel_output_shift, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output)); -} - -void EvalQuantized(TfLiteContext* context, TfLiteNode* node, - TfLiteDepthwiseConvParams* params, OpData* data, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* output) { - const int32_t input_offset = -input->params.zero_point; - const int32_t filter_offset = -filter->params.zero_point; - const int32_t output_offset = output->params.zero_point; - - tflite::DepthwiseParams op_params; - // Padding type is ignored, but still set. - op_params.padding_type = PaddingType::kSame; - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; - op_params.stride_width = params->stride_width; - op_params.stride_height = params->stride_height; - op_params.dilation_width_factor = params->dilation_width_factor; - op_params.dilation_height_factor = params->dilation_height_factor; - op_params.depth_multiplier = params->depth_multiplier; - op_params.quantized_activation_min = data->output_activation_min; - op_params.quantized_activation_max = data->output_activation_max; - op_params.input_offset = input_offset; - op_params.weights_offset = filter_offset; - op_params.output_offset = output_offset; - op_params.output_multiplier = data->output_multiplier; - // Legacy ops used mixed left and right shifts. Now all are +ve-means-left. - op_params.output_shift = -data->output_shift; - - tflite::reference_ops::DepthwiseConv( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(filter), GetTensorData(filter), - GetTensorShape(bias), GetTensorData(bias), - GetTensorShape(output), GetTensorData(output)); -} - -TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { auto* params = reinterpret_cast(node->builtin_data); + OpData* data = static_cast(node->user_data); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); - const TfLiteTensor* bias = - (NumInputs(node) == 3) ? GetInput(context, node, kBiasTensor) : nullptr; + TF_LITE_ENSURE(context, filter != nullptr); const TfLiteType data_type = input->type; int width = SizeOfDimension(input, 2); @@ -204,7 +124,16 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { int filter_width = SizeOfDimension(filter, 2); int filter_height = SizeOfDimension(filter, 1); - OpData data; + // Per channel quantization is only needed for int8_t inference. For other + // quantized types, only a single scale and zero point is needed. + const int num_channels = filter->dims->data[kDepthwiseConvQuantizedDimension]; + // Dynimically allocate per-channel quantization parameters. + data->per_channel_output_multiplier = + reinterpret_cast(context->AllocatePersistentBuffer( + context, num_channels * sizeof(int32_t))); + data->per_channel_output_shift = + reinterpret_cast(context->AllocatePersistentBuffer( + context, num_channels * sizeof(int32_t))); // All per-channel quantized tensors need valid zero point and scale arrays. if (input->type == kTfLiteInt8) { @@ -227,20 +156,151 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_STATUS(CalculateOpData(context, node, params, width, height, filter_width, filter_height, data_type, - &data)); + data)); + + data->input_zero_point = input->params.zero_point; + data->filter_zero_point = filter->params.zero_point; + data->output_zero_point = output->params.zero_point; + + return kTfLiteOk; +} + +void EvalFloat(TfLiteContext* context, TfLiteNode* node, + TfLiteDepthwiseConvParams* params, const OpData& data, + const TfLiteEvalTensor* input, const TfLiteEvalTensor* filter, + const TfLiteEvalTensor* bias, TfLiteEvalTensor* output) { + float output_activation_min, output_activation_max; + CalculateActivationRange(params->activation, &output_activation_min, + &output_activation_max); + + tflite::DepthwiseParams op_params; + // Padding type is ignored, but still set. + op_params.padding_type = PaddingType::kSame; + op_params.padding_values.width = data.padding.width; + op_params.padding_values.height = data.padding.height; + op_params.stride_width = params->stride_width; + op_params.stride_height = params->stride_height; + op_params.dilation_width_factor = params->dilation_width_factor; + op_params.dilation_height_factor = params->dilation_height_factor; + op_params.depth_multiplier = params->depth_multiplier; + op_params.float_activation_min = output_activation_min; + op_params.float_activation_max = output_activation_max; + + tflite::reference_ops::DepthwiseConv( + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +} + +void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, + TfLiteDepthwiseConvParams* params, + const OpData& data, const TfLiteEvalTensor* input, + const TfLiteEvalTensor* filter, + const TfLiteEvalTensor* bias, + TfLiteEvalTensor* output) { + DepthwiseParams op_params; + op_params.padding_type = PaddingType::kSame; + op_params.padding_values.width = data.padding.width; + op_params.padding_values.height = data.padding.height; + op_params.stride_width = params->stride_width; + op_params.stride_height = params->stride_height; + op_params.dilation_width_factor = params->dilation_width_factor; + op_params.dilation_height_factor = params->dilation_height_factor; + op_params.depth_multiplier = params->depth_multiplier; + op_params.input_offset = -data.input_zero_point; + op_params.weights_offset = 0; + op_params.output_offset = data.output_zero_point; + // TODO(b/130439627): Use calculated value for clamping. + op_params.quantized_activation_min = std::numeric_limits::min(); + op_params.quantized_activation_max = std::numeric_limits::max(); + + reference_integer_ops::DepthwiseConvPerChannel( + op_params, data.per_channel_output_multiplier, + data.per_channel_output_shift, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +} + +void EvalQuantized(TfLiteContext* context, TfLiteNode* node, + TfLiteDepthwiseConvParams* params, const OpData& data, + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* filter, const TfLiteEvalTensor* bias, + TfLiteEvalTensor* output) { + const int32_t input_offset = -data.input_zero_point; + const int32_t filter_offset = -data.filter_zero_point; + const int32_t output_offset = data.output_zero_point; + + tflite::DepthwiseParams op_params; + // Padding type is ignored, but still set. + op_params.padding_type = PaddingType::kSame; + op_params.padding_values.width = data.padding.width; + op_params.padding_values.height = data.padding.height; + op_params.stride_width = params->stride_width; + op_params.stride_height = params->stride_height; + op_params.dilation_width_factor = params->dilation_width_factor; + op_params.dilation_height_factor = params->dilation_height_factor; + op_params.depth_multiplier = params->depth_multiplier; + op_params.quantized_activation_min = data.output_activation_min; + op_params.quantized_activation_max = data.output_activation_max; + op_params.input_offset = input_offset; + op_params.weights_offset = filter_offset; + op_params.output_offset = output_offset; + op_params.output_multiplier = data.output_multiplier; + // Legacy ops used mixed left and right shifts. Now all are +ve-means-left. + op_params.output_shift = -data.output_shift; + + tflite::reference_ops::DepthwiseConv( + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); + + auto* params = + reinterpret_cast(node->builtin_data); + const OpData& data = *(static_cast(node->user_data)); + + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + const TfLiteEvalTensor* filter = + tflite::micro::GetEvalInput(context, node, kFilterTensor); + const TfLiteEvalTensor* bias = + (NumInputs(node) == 3) + ? tflite::micro::GetEvalInput(context, node, kBiasTensor) + : nullptr; // TODO(aselle): Consider whether float conv and quantized conv should be // separate ops to avoid dispatch overhead here. switch (input->type) { // Already know in/out types are same. case kTfLiteFloat32: - EvalFloat(context, node, params, &data, input, filter, bias, output); + EvalFloat(context, node, params, data, input, filter, bias, output); break; case kTfLiteInt8: - EvalQuantizedPerChannel(context, node, params, &data, input, filter, bias, + EvalQuantizedPerChannel(context, node, params, data, input, filter, bias, output); break; case kTfLiteUInt8: - EvalQuantized(context, node, params, &data, input, filter, bias, output); + EvalQuantized(context, node, params, data, input, filter, bias, output); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -250,20 +310,17 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } -} // namespace depthwise_conv +} // namespace -TfLiteRegistration* Register_DEPTHWISE_CONV_2D() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/depthwise_conv::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_DEPTHWISE_CONV_2D() { + return {/*init=*/Init, + /*free=*/nullptr, + /*prepare=*/Prepare, + /*invoke=*/Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -} // namespace micro -} // namespace ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/dequantize.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/dequantize.cc index 37fb8ffc..f4e2eb9f 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/dequantize.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/dequantize.cc @@ -22,19 +22,39 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/reference/requantize.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { namespace micro { namespace dequantize { +struct OpData { + tflite::DequantizationParams quantization_params; + // The scaling factor from input to output (aka the 'real multiplier') can + // be represented as a fixed point multiplier plus a left shift. + int32_t output_multiplier; + int output_shift; + int32_t output_zero_point; +}; + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); // TODO(b/140515557): Add cached dequant to improve hybrid model performance. const TfLiteTensor* input = GetInput(context, node, 0); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, 0); + TF_LITE_ENSURE(context, output != nullptr); TF_LITE_ENSURE(context, input->type == kTfLiteUInt8 || input->type == kTfLiteInt8 || @@ -42,32 +62,49 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE( context, output->type == kTfLiteFloat32 || output->type == kTfLiteInt32); + if (output->type == kTfLiteInt32) { + const double effective_output_scale = + static_cast(input->params.scale) / + static_cast(output->params.scale); + QuantizeMultiplier(effective_output_scale, &data->output_multiplier, + &data->output_shift); + } + + data->quantization_params.zero_point = input->params.zero_point; + data->quantization_params.scale = static_cast(input->params.scale); + data->output_zero_point = output->params.zero_point; return kTfLiteOk; } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, 0); - TfLiteTensor* output = GetOutput(context, node, 0); + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); if (output->type == kTfLiteFloat32) { - tflite::DequantizationParams op_params; - op_params.zero_point = input->params.zero_point; - op_params.scale = static_cast(input->params.scale); switch (input->type) { case kTfLiteUInt8: - reference_ops::Dequantize( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + reference_ops::Dequantize(data->quantization_params, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; case kTfLiteInt8: - reference_ops::Dequantize( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + reference_ops::Dequantize(data->quantization_params, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; case kTfLiteInt16: - reference_ops::Dequantize( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + reference_ops::Dequantize(data->quantization_params, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; default: TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", @@ -76,28 +113,23 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteError; } } else if (output->type == kTfLiteInt32) { - int32_t output_multiplier; - int output_shift; - const double effective_output_scale = - static_cast(input->params.scale) / - static_cast(output->params.scale); - QuantizeMultiplier(effective_output_scale, &output_multiplier, - &output_shift); - int flat_size = - MatchingFlatSize(GetTensorShape(input), GetTensorShape(output)); + int flat_size = MatchingFlatSize(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorShape(output)); switch (input->type) { case kTfLiteInt16: { reference_ops::Requantize( - GetTensorData(input), flat_size, output_multiplier, - output_shift, input->params.zero_point, output->params.zero_point, - GetTensorData(output)); + tflite::micro::GetTensorData(input), flat_size, + data->output_multiplier, data->output_shift, + data->quantization_params.zero_point, data->output_zero_point, + tflite::micro::GetTensorData(output)); break; } case kTfLiteInt8: { reference_ops::Requantize( - GetTensorData(input), flat_size, output_multiplier, - output_shift, input->params.zero_point, output->params.zero_point, - GetTensorData(output)); + tflite::micro::GetTensorData(input), flat_size, + data->output_multiplier, data->output_shift, + data->quantization_params.zero_point, data->output_zero_point, + tflite::micro::GetTensorData(output)); break; } default: @@ -118,16 +150,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace dequantize -TfLiteRegistration* Register_DEQUANTIZE() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/dequantize::Prepare, - /*invoke=*/dequantize::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_DEQUANTIZE() { + return {/*init=*/dequantize::Init, + /*free=*/nullptr, + /*prepare=*/dequantize::Prepare, + /*invoke=*/dequantize::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/elementwise.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/elementwise.cc index 2e3ebe9f..581e532b 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/elementwise.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/elementwise.cc @@ -18,6 +18,8 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/micro_utils.h" namespace tflite { namespace ops { @@ -39,8 +41,10 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); const TfLiteTensor* input = GetInput(context, node, 0); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, 0); - TF_LITE_ENSURE_EQ(context, input->type, output->type); + TF_LITE_ENSURE(context, output != nullptr); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); if (!IsSupportedType(input->type)) { TF_LITE_KERNEL_LOG(context, "Input data type %s (%d) is not supported.", TfLiteTypeGetName(input->type), input->type); @@ -52,13 +56,13 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { template inline TfLiteStatus EvalImpl(TfLiteContext* context, TfLiteNode* node, T func(T), TfLiteType expected_type) { - const TfLiteTensor* input = GetInput(context, node, 0); - TfLiteTensor* output = GetOutput(context, node, 0); - TF_LITE_ENSURE_EQ(context, input->type, expected_type); - const int64_t num_elements = NumElements(input); - const T* in_data = GetTensorData(input); - T* out_data = GetTensorData(output); - for (int64_t i = 0; i < num_elements; ++i) { + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, expected_type); + const size_t num_elements = ElementCount(*input->dims); + const T* in_data = tflite::micro::GetTensorData(input); + T* out_data = tflite::micro::GetTensorData(output); + for (size_t i = 0; i < num_elements; ++i) { out_data[i] = func(in_data[i]); } return kTfLiteOk; @@ -109,116 +113,100 @@ TfLiteStatus LogicalNotEval(TfLiteContext* context, TfLiteNode* node) { } // namespace } // namespace elementwise -TfLiteRegistration* Register_ABS() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::AbsEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_ABS() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ + elementwise::GenericPrepare, + /*invoke=*/elementwise::AbsEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_SIN() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::SinEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_SIN() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ + elementwise::GenericPrepare, + /*invoke=*/elementwise::SinEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_COS() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::CosEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_COS() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ + elementwise::GenericPrepare, + /*invoke=*/elementwise::CosEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_LOG() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::LogEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_LOG() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ + elementwise::GenericPrepare, + /*invoke=*/elementwise::LogEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_SQRT() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::SqrtEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_SQRT() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ + elementwise::GenericPrepare, + /*invoke=*/elementwise::SqrtEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_RSQRT() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::RsqrtEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_RSQRT() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ + elementwise::GenericPrepare, + /*invoke=*/elementwise::RsqrtEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_SQUARE() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::SquareEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_SQUARE() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ + elementwise::GenericPrepare, + /*invoke=*/elementwise::SquareEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_LOGICAL_NOT() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::LogicalNotEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_LOGICAL_NOT() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/ + elementwise::GenericPrepare, + /*invoke=*/elementwise::LogicalNotEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/string_type.h b/code/lib/tfmicro/tensorflow/lite/micro/kernels/ethosu.cc similarity index 63% rename from code/lib/tfmicro/tensorflow/lite/string_type.h rename to code/lib/tfmicro/tensorflow/lite/micro/kernels/ethosu.cc index f5a7f833..eac6cea8 100644 --- a/code/lib/tfmicro/tensorflow/lite/string_type.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/ethosu.cc @@ -1,4 +1,4 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,21 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// Abstract string. We don't want even absl at this level. -#ifndef TENSORFLOW_LITE_STRING_TYPE_H_ -#define TENSORFLOW_LITE_STRING_TYPE_H_ -#include +// +// This is a stub file for non-Ethos platforms +// +#include "tensorflow/lite/c/common.h" namespace tflite { +namespace ops { +namespace micro { +namespace custom { +TfLiteRegistration* Register_ETHOSU() { return nullptr; } -using std::string; +const char* GetString_ETHOSU() { return ""; } +} // namespace custom +} // namespace micro +} // namespace ops } // namespace tflite - -#endif // TENSORFLOW_LITE_STRING_TYPE_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/floor.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/floor.cc index 435934fe..b8be1cf0 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/floor.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/floor.cc @@ -17,7 +17,7 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" -#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -28,25 +28,28 @@ constexpr int kInputTensor = 0; constexpr int kOutputTensor = 0; TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - reference_ops::Floor(GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, kTfLiteFloat32); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + reference_ops::Floor(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } } // namespace floor -TfLiteRegistration* Register_FLOOR() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/floor::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_FLOOR() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/floor::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/fully_connected.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/fully_connected.cc index 66b83797..d3fdeacb 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/fully_connected.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/fully_connected.cc @@ -1,4 +1,4 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,20 +13,19 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/lite/kernels/internal/reference/fully_connected.h" +#include "tensorflow/lite/micro/kernels/fully_connected.h" #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/fully_connected.h" #include "tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { -namespace ops { -namespace micro { -namespace fully_connected { namespace { struct OpData { @@ -40,6 +39,10 @@ struct OpData { int32_t output_activation_max; // The index of the temporary tensor where the quantized inputs are cached. int input_quantized_index; + // Cached zero point values of tensors. + int32_t input_zero_point; + int32_t filter_zero_point; + int32_t output_zero_point; }; constexpr int kInputTensor = 0; @@ -64,20 +67,17 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TF_LITE_ENSURE_STATUS(CalculateActivationRangeQuantized( context, activation, output, &data->output_activation_min, &data->output_activation_max)); + + data->input_zero_point = input->params.zero_point; + data->filter_zero_point = filter->params.zero_point; + data->output_zero_point = output->params.zero_point; } return status; } -} // namespace - void* Init(TfLiteContext* context, const char* buffer, size_t length) { TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); - void* data = nullptr; - if (context->AllocatePersistentBuffer(context, sizeof(OpData), &data) == - kTfLiteError) { - return nullptr; - } - return data; + return context->AllocatePersistentBuffer(context, sizeof(OpData)); } TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { @@ -89,11 +89,14 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { static_cast(node->builtin_data); const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); const TfLiteTensor* filter = GetInput(context, node, kWeightsTensor); + TF_LITE_ENSURE(context, filter != nullptr); const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); - TF_LITE_ENSURE_EQ(context, input->type, output->type); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); TF_LITE_ENSURE_MSG(context, input->type == filter->type, "Hybrid models are not supported on TFLite Micro."); @@ -102,13 +105,15 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus EvalQuantizedInt8(TfLiteContext* context, TfLiteNode* node, - const OpData& data, const TfLiteTensor* input, - const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* output) { + const OpData& data, + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* filter, + const TfLiteEvalTensor* bias, + TfLiteEvalTensor* output) { tflite::FullyConnectedParams op_params; - op_params.input_offset = -input->params.zero_point; - op_params.weights_offset = -filter->params.zero_point; - op_params.output_offset = output->params.zero_point; + op_params.input_offset = -data.input_zero_point; + op_params.weights_offset = -data.filter_zero_point; + op_params.output_offset = data.output_zero_point; op_params.output_multiplier = data.output_multiplier; // TODO(b/138810107): Figure out whether output shift should be inverted op_params.output_shift = -data.output_shift; @@ -116,20 +121,25 @@ TfLiteStatus EvalQuantizedInt8(TfLiteContext* context, TfLiteNode* node, op_params.quantized_activation_max = data.output_activation_max; reference_integer_ops::FullyConnected( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(filter), GetTensorData(filter), - GetTensorShape(bias), GetTensorData(bias), - GetTensorShape(output), GetTensorData(output)); + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } TfLiteStatus EvalQuantized(TfLiteContext* context, TfLiteNode* node, - const OpData& data, const TfLiteTensor* input, - const TfLiteTensor* filter, const TfLiteTensor* bias, - TfLiteTensor* output) { - const int32_t input_offset = -input->params.zero_point; - const int32_t filter_offset = -filter->params.zero_point; - const int32_t output_offset = output->params.zero_point; + const OpData& data, const TfLiteEvalTensor* input, + const TfLiteEvalTensor* filter, + const TfLiteEvalTensor* bias, + TfLiteEvalTensor* output) { + const int32_t input_offset = -data.input_zero_point; + const int32_t filter_offset = -data.filter_zero_point; + const int32_t output_offset = data.output_zero_point; tflite::FullyConnectedParams op_params; op_params.input_offset = input_offset; @@ -141,12 +151,16 @@ TfLiteStatus EvalQuantized(TfLiteContext* context, TfLiteNode* node, op_params.quantized_activation_min = data.output_activation_min; op_params.quantized_activation_max = data.output_activation_max; -#define TF_LITE_FULLY_CONNECTED(output_data_type) \ - reference_ops::FullyConnected( \ - op_params, GetTensorShape(input), GetTensorData(input), \ - GetTensorShape(filter), GetTensorData(filter), \ - GetTensorShape(bias), GetTensorData(bias), \ - GetTensorShape(output), GetTensorData(output)) +#define TF_LITE_FULLY_CONNECTED(output_data_type) \ + reference_ops::FullyConnected( \ + op_params, tflite::micro::GetTensorShape(input), \ + tflite::micro::GetTensorData(input), \ + tflite::micro::GetTensorShape(filter), \ + tflite::micro::GetTensorData(filter), \ + tflite::micro::GetTensorShape(bias), \ + tflite::micro::GetTensorData(bias), \ + tflite::micro::GetTensorShape(output), \ + tflite::micro::GetTensorData(output)) switch (output->type) { case kTfLiteUInt8: TF_LITE_FULLY_CONNECTED(uint8_t); @@ -165,8 +179,9 @@ TfLiteStatus EvalQuantized(TfLiteContext* context, TfLiteNode* node, TfLiteStatus EvalFloat(TfLiteContext* context, TfLiteNode* node, TfLiteFusedActivation activation, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* output) { + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* filter, + const TfLiteEvalTensor* bias, TfLiteEvalTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRange(activation, &output_activation_min, &output_activation_max); @@ -174,10 +189,14 @@ TfLiteStatus EvalFloat(TfLiteContext* context, TfLiteNode* node, op_params.float_activation_min = output_activation_min; op_params.float_activation_max = output_activation_max; tflite::reference_ops::FullyConnected( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(filter), GetTensorData(filter), - GetTensorShape(bias), GetTensorData(bias), GetTensorShape(output), - GetTensorData(output)); + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } @@ -186,10 +205,14 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const auto* params = static_cast(node->builtin_data); - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - const TfLiteTensor* filter = GetInput(context, node, kWeightsTensor); - const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + const TfLiteEvalTensor* filter = + tflite::micro::GetEvalInput(context, node, kWeightsTensor); + const TfLiteEvalTensor* bias = + tflite::micro::GetEvalInput(context, node, kBiasTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); TFLITE_DCHECK(node->user_data != nullptr); const OpData& data = *(static_cast(node->user_data)); @@ -214,20 +237,17 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } -} // namespace fully_connected +} // namespace -TfLiteRegistration* Register_FULLY_CONNECTED() { - static TfLiteRegistration r = {/*init=*/fully_connected::Init, - /*free=*/nullptr, - /*prepare=*/fully_connected::Prepare, - /*invoke=*/fully_connected::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_FULLY_CONNECTED() { + return {/*init=*/Init, + /*free=*/nullptr, + /*prepare=*/Prepare, + /*invoke=*/Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -} // namespace micro -} // namespace ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/fully_connected.h b/code/lib/tfmicro/tensorflow/lite/micro/kernels/fully_connected.h new file mode 100644 index 00000000..3e646718 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/fully_connected.h @@ -0,0 +1,50 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_FULLY_CONNECTED_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_FULLY_CONNECTED_H_ + +#include "tensorflow/lite/c/common.h" + +namespace tflite { + +// This is the most generic TfLiteRegistration. The actual supported types may +// still be target dependent. The only requirement is that every implementation +// (reference or optimized) must define this function. +TfLiteRegistration Register_FULLY_CONNECTED(); + +#if defined(CMSIS_NN) || defined(ARDUINO) +// The Arduino is a special case where we use the CMSIS kernels, but because of +// the current approach to building for Arduino, we do not support -DCMSIS_NN as +// part of the build. As a result, we use defined(ARDUINO) as proxy for the +// CMSIS kernels for this one special case. + +// Returns a TfLiteRegistration struct for cmsis-nn kernel variant that only +// supports int8. +TfLiteRegistration Register_FULLY_CONNECTED_INT8(); + +#else +// Note that while this block gets used for both reference and optimized kernels +// that do not have any specialized implementations, the only goal here is to +// define fallback implementation that allow reference kernels to still be used +// from applications that call a more specific kernel variant. + +inline TfLiteRegistration Register_FULLY_CONNECTED_INT8() { + return Register_FULLY_CONNECTED(); +} + +#endif +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_KERNELS_FULLY_CONNECTED_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/hard_swish.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/hard_swish.cc new file mode 100644 index 00000000..a0a245f8 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/hard_swish.cc @@ -0,0 +1,142 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/kernels/internal/reference/hard_swish.h" + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/internal/types.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/micro_utils.h" + +namespace tflite { +namespace ops { +namespace micro { +namespace hard_swish { + +constexpr int kInputTensor = 0; +constexpr int kOutputTensor = 0; + +void* HardSwishInit(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(HardSwishParams)); +} + +TfLiteStatus HardSwishPrepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + if (input->type == kTfLiteUInt8 || input->type == kTfLiteInt8) { + HardSwishParams* params = static_cast(node->user_data); + + params->input_zero_point = input->params.zero_point; + params->output_zero_point = output->params.zero_point; + + const float input_scale = input->params.scale; + const float hires_input_scale = (1.0f / 128.0f) * input_scale; + const float reluish_scale = 3.0f / 32768.0f; + const float output_scale = output->params.scale; + + const double output_multiplier = + static_cast(hires_input_scale / output_scale); + int32_t output_multiplier_fixedpoint_int32; + QuantizeMultiplier(output_multiplier, &output_multiplier_fixedpoint_int32, + ¶ms->output_multiplier_exponent); + DownScaleInt32ToInt16Multiplier( + output_multiplier_fixedpoint_int32, + ¶ms->output_multiplier_fixedpoint_int16); + + TF_LITE_ENSURE(context, params->output_multiplier_exponent <= 0); + + const double reluish_multiplier = + static_cast(hires_input_scale / reluish_scale); + int32_t reluish_multiplier_fixedpoint_int32; + QuantizeMultiplier(reluish_multiplier, &reluish_multiplier_fixedpoint_int32, + ¶ms->reluish_multiplier_exponent); + DownScaleInt32ToInt16Multiplier( + reluish_multiplier_fixedpoint_int32, + ¶ms->reluish_multiplier_fixedpoint_int16); + } + + return kTfLiteOk; +} + +TfLiteStatus HardSwishEval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + HardSwishParams* params = static_cast(node->user_data); + + switch (input->type) { + case kTfLiteFloat32: { + tflite::reference_ops::HardSwish( + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } break; + case kTfLiteUInt8: { + tflite::reference_ops::HardSwish( + *params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } break; + case kTfLiteInt8: { + tflite::reference_ops::HardSwish( + *params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } break; + default: { + TF_LITE_KERNEL_LOG( + context, + "Only float32/int8_t/uint8_t are supported currently, got %s", + TfLiteTypeGetName(input->type)); + return kTfLiteError; + } + } + return kTfLiteOk; +} + +} // namespace hard_swish + +TfLiteRegistration Register_HARD_SWISH() { + return {/*init=*/hard_swish::HardSwishInit, + /*free=*/nullptr, + /*prepare=*/hard_swish::HardSwishPrepare, + /*invoke=*/hard_swish::HardSwishEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; +} + +} // namespace micro +} // namespace ops +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_runner.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_runner.cc new file mode 100644 index 00000000..cef6c01c --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_runner.cc @@ -0,0 +1,165 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/kernels/kernel_runner.h" + +namespace tflite { +namespace micro { + +namespace { +constexpr size_t kBufferAlignment = 16; +} // namespace + +// TODO(b/161841696): Consider moving away from global arena buffers: +constexpr int KernelRunner::kNumScratchBuffers_; +constexpr int KernelRunner::kKernelRunnerBufferSize_; +uint8_t KernelRunner::kKernelRunnerBuffer_[]; + +KernelRunner::KernelRunner(const TfLiteRegistration& registration, + TfLiteTensor* tensors, int tensors_size, + TfLiteIntArray* inputs, TfLiteIntArray* outputs, + void* builtin_data, ErrorReporter* error_reporter) + : allocator_(SimpleMemoryAllocator::Create( + error_reporter, kKernelRunnerBuffer_, kKernelRunnerBufferSize_)), + registration_(registration), + tensors_(tensors), + error_reporter_(error_reporter) { + // Prepare TfLiteContext: + context_.impl_ = static_cast(this); + context_.ReportError = ReportOpError; + context_.recommended_num_threads = 1; + context_.GetTensor = GetTensor; + context_.GetEvalTensor = GetEvalTensor; + context_.AllocatePersistentBuffer = AllocatePersistentBuffer; + context_.RequestScratchBufferInArena = RequestScratchBufferInArena; + context_.GetScratchBuffer = GetScratchBuffer; + + // Prepare TfLiteNode: + node_.inputs = inputs; + node_.outputs = outputs; + node_.builtin_data = builtin_data; +} + +TfLiteStatus KernelRunner::InitAndPrepare(const char* init_data) { + if (registration_.init) { + node_.user_data = registration_.init(&context_, init_data, /*length=*/0); + } + if (registration_.prepare) { + TF_LITE_ENSURE_STATUS(registration_.prepare(&context_, &node_)); + } + return kTfLiteOk; +} + +TfLiteStatus KernelRunner::Invoke() { + if (registration_.invoke == nullptr) { + TF_LITE_REPORT_ERROR(error_reporter_, + "TfLiteRegistration missing invoke function pointer!"); + return kTfLiteError; + } + return registration_.invoke(&context_, &node_); +} + +TfLiteTensor* KernelRunner::GetTensor(const struct TfLiteContext* context, + int tensor_index) { + TFLITE_DCHECK(context != nullptr); + KernelRunner* runner = reinterpret_cast(context->impl_); + TFLITE_DCHECK(runner != nullptr); + + return &runner->tensors_[tensor_index]; +} + +TfLiteEvalTensor* KernelRunner::GetEvalTensor( + const struct TfLiteContext* context, int tensor_index) { + TFLITE_DCHECK(context != nullptr); + KernelRunner* runner = reinterpret_cast(context->impl_); + TFLITE_DCHECK(runner != nullptr); + + TfLiteEvalTensor* eval_tensor = + reinterpret_cast(runner->allocator_->AllocateTemp( + sizeof(TfLiteEvalTensor), alignof(TfLiteEvalTensor))); + TFLITE_DCHECK(eval_tensor != nullptr); + + // In unit tests, the TfLiteTensor pointer contains the source of truth for + // buffers and values: + eval_tensor->data = runner->tensors_[tensor_index].data; + eval_tensor->dims = runner->tensors_[tensor_index].dims; + eval_tensor->type = runner->tensors_[tensor_index].type; + return eval_tensor; +} + +void* KernelRunner::AllocatePersistentBuffer(TfLiteContext* context, + size_t bytes) { + TFLITE_DCHECK(context != nullptr); + KernelRunner* runner = reinterpret_cast(context->impl_); + TFLITE_DCHECK(runner != nullptr); + + return runner->allocator_->AllocateFromTail(bytes, kBufferAlignment); +} + +TfLiteStatus KernelRunner::RequestScratchBufferInArena(TfLiteContext* context, + size_t bytes, + int* buffer_index) { + TFLITE_DCHECK(context != nullptr); + TFLITE_DCHECK(buffer_index != nullptr); + + KernelRunner* runner = reinterpret_cast(context->impl_); + TFLITE_DCHECK(runner != nullptr); + + if (runner->scratch_buffer_count_ == kNumScratchBuffers_) { + TF_LITE_REPORT_ERROR( + runner->error_reporter_, + "Exceeded the maximum number of scratch tensors allowed (%d).", + kNumScratchBuffers_); + return kTfLiteError; + } + + // For tests, we allocate scratch buffers from the tail and keep them around + // for the lifetime of model. This means that the arena size in the tests will + // be more than what we would have if the scratch buffers could share memory. + runner->scratch_buffers_[runner->scratch_buffer_count_] = + runner->allocator_->AllocateFromTail(bytes, kBufferAlignment); + TFLITE_DCHECK(runner->scratch_buffers_[runner->scratch_buffer_count_] != + nullptr); + + *buffer_index = runner->scratch_buffer_count_++; + return kTfLiteOk; +} + +void* KernelRunner::GetScratchBuffer(TfLiteContext* context, int buffer_index) { + TFLITE_DCHECK(context != nullptr); + KernelRunner* runner = reinterpret_cast(context->impl_); + TFLITE_DCHECK(runner != nullptr); + + TFLITE_DCHECK(runner->scratch_buffer_count_ <= kNumScratchBuffers_); + if (buffer_index >= runner->scratch_buffer_count_) { + return nullptr; + } + return runner->scratch_buffers_[buffer_index]; +} + +void KernelRunner::ReportOpError(struct TfLiteContext* context, + const char* format, ...) { + TFLITE_DCHECK(context != nullptr); + KernelRunner* runner = reinterpret_cast(context->impl_); + TFLITE_DCHECK(runner != nullptr); + + va_list args; + va_start(args, format); + TF_LITE_REPORT_ERROR(runner->error_reporter_, format, args); + va_end(args); +} + +} // namespace micro +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_runner.h b/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_runner.h new file mode 100644 index 00000000..45d107e7 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_runner.h @@ -0,0 +1,83 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_KERNEL_RUNNER_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_KERNEL_RUNNER_H_ + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/simple_memory_allocator.h" + +namespace tflite { +namespace micro { + +// Helper class to perform a simulated kernel (i.e. TfLiteRegistration) lifecyle +// (init, prepare, invoke). All internal allocations are handled by this class. +// Simply pass in the registration, list of required tensors, inputs array, +// outputs array, and any pre-builtin data. Calling Invoke() will automatically +// walk the kernl and outputs will be ready on the the TfLiteTensor output +// provided during construction. +class KernelRunner { + public: + KernelRunner(const TfLiteRegistration& registration, TfLiteTensor* tensors, + int tensors_size, TfLiteIntArray* inputs, + TfLiteIntArray* outputs, void* builtin_data, + ErrorReporter* error_reporter); + + // Calls init and prepare on the kernel (i.e. TfLiteRegistration) struct. Any + // exceptions will be reported through the error_reporter and returned as a + // status code here. + TfLiteStatus InitAndPrepare(const char* init_data = nullptr); + + // Calls init, prepare, and invoke on a given TfLiteRegistration pointer. + // After successful invoke, results will be available in the output tensor as + // passed into the constructor of this class. + TfLiteStatus Invoke(); + + protected: + static TfLiteTensor* GetTensor(const struct TfLiteContext* context, + int tensor_index); + static TfLiteEvalTensor* GetEvalTensor(const struct TfLiteContext* context, + int tensor_index); + static void* AllocatePersistentBuffer(TfLiteContext* context, size_t bytes); + static TfLiteStatus RequestScratchBufferInArena(TfLiteContext* context, + size_t bytes, + int* buffer_index); + static void* GetScratchBuffer(TfLiteContext* context, int buffer_index); + static void ReportOpError(struct TfLiteContext* context, const char* format, + ...); + + private: + static constexpr int kNumScratchBuffers_ = 5; + + static constexpr int kKernelRunnerBufferSize_ = 10000; + static uint8_t kKernelRunnerBuffer_[kKernelRunnerBufferSize_]; + + SimpleMemoryAllocator* allocator_ = nullptr; + const TfLiteRegistration& registration_; + TfLiteTensor* tensors_ = nullptr; + ErrorReporter* error_reporter_ = nullptr; + + TfLiteContext context_ = {}; + TfLiteNode node_ = {}; + + int scratch_buffer_count_ = 0; + uint8_t* scratch_buffers_[kNumScratchBuffers_]; +}; + +} // namespace micro +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_KERNELS_KERNEL_RUNNER_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_util.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_util.cc new file mode 100644 index 00000000..deca92b6 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_util.cc @@ -0,0 +1,41 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/kernels/kernel_util.h" + +#include "tensorflow/lite/c/common.h" + +namespace tflite { +namespace micro { + +bool HaveSameShapes(const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2) { + TFLITE_DCHECK(input1 != nullptr); + TFLITE_DCHECK(input2 != nullptr); + return TfLiteIntArrayEqual(input1->dims, input2->dims); +} + +const RuntimeShape GetTensorShape(const TfLiteEvalTensor* tensor) { + if (tensor == nullptr || tensor->dims == nullptr) { + return RuntimeShape(); + } + TfLiteIntArray* dims = tensor->dims; + const int dims_size = dims->size; + const int32_t* dims_data = reinterpret_cast(dims->data); + return RuntimeShape(dims_size, dims_data); +} + +} // namespace micro +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_util.h b/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_util.h new file mode 100644 index 00000000..79cd58ec --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/kernel_util.h @@ -0,0 +1,75 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_KERNEL_UTIL_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_KERNEL_UTIL_H_ + +#include + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/kernels/internal/types.h" + +namespace tflite { +namespace micro { + +// Returns a mutable tensor for a given input index. is_variable must be checked +// during prepare when the full TfLiteTensor is available. +inline TfLiteEvalTensor* GetMutableEvalInput(const TfLiteContext* context, + const TfLiteNode* node, + int index) { + TFLITE_DCHECK(context != nullptr); + TFLITE_DCHECK(node != nullptr); + return context->GetEvalTensor(context, node->inputs->data[index]); +} + +// Returns the TfLiteEvalTensor struct for a given input index in a node. +inline const TfLiteEvalTensor* GetEvalInput(const TfLiteContext* context, + const TfLiteNode* node, int index) { + return GetMutableEvalInput(context, node, index); +} + +// Returns the TfLiteEvalTensor struct for a given output index in a node. +inline TfLiteEvalTensor* GetEvalOutput(const TfLiteContext* context, + const TfLiteNode* node, int index) { + TFLITE_DCHECK(context != nullptr); + TFLITE_DCHECK(node != nullptr); + return context->GetEvalTensor(context, node->outputs->data[index]); +} + +// Returns data for a TfLiteEvalTensor struct. +template +T* GetTensorData(TfLiteEvalTensor* tensor) { + return tensor != nullptr ? reinterpret_cast(tensor->data.raw) : nullptr; +} + +// Returns const data for a TfLiteEvalTensor struct. +template +const T* GetTensorData(const TfLiteEvalTensor* tensor) { + TFLITE_DCHECK(tensor != nullptr); + return reinterpret_cast(tensor->data.raw); +} + +// Returns the shape of a TfLiteEvalTensor struct. +const RuntimeShape GetTensorShape(const TfLiteEvalTensor* tensor); + +// Return true if the given tensors have the same shape. +bool HaveSameShapes(const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2); + +} // namespace micro +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_KERNELS_KERNEL_UTIL_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/l2norm.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/l2norm.cc index 4dd71fe1..401741a0 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/l2norm.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/l2norm.cc @@ -14,16 +14,19 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/portable_tensor.h" #include "tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h" #include "tensorflow/lite/kernels/internal/reference/l2normalization.h" -#include "tensorflow/lite/kernels/internal/tensor.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { namespace micro { namespace l2norm { +namespace { + // This file has two implementation of L2Norm. enum KernelType { kReference, @@ -33,44 +36,59 @@ enum KernelType { constexpr int kInputTensor = 0; constexpr int kOutputTensor = 0; +} // namespace + TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { -#if defined(DEBUG) + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); + auto* params = reinterpret_cast(node->builtin_data); + L2NormalizationParams* data = + static_cast(node->user_data); TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); TF_LITE_ENSURE(context, NumDimensions(input) <= 4); TF_LITE_ENSURE(context, output->type == kTfLiteFloat32 || output->type == kTfLiteUInt8 || output->type == kTfLiteInt8); - TF_LITE_ENSURE_EQ(context, input->type, output->type); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8) { - TF_LITE_ENSURE_EQ(context, output->params.scale, (1. / 128.)); - if (output->type == kTfLiteUInt8) { - TF_LITE_ENSURE_EQ(context, output->params.zero_point, 128); - } - if (output->type == kTfLiteInt8) { - TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); - } + data->input_zero_point = input->params.zero_point; + } else if (output->type == kTfLiteFloat32) { + data->input_zero_point = 0; } // TODO(ahentz): For some reason our implementations don't support // activations. TF_LITE_ENSURE_EQ(context, params->activation, kTfLiteActNone); -#endif return kTfLiteOk; } +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, + sizeof(L2NormalizationParams)); +} + TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TFLITE_DCHECK(node->user_data != nullptr); + const L2NormalizationParams& data = + *(static_cast(node->user_data)); + + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); // TODO(b/143912164): instead of hardcode the epsilon here, we should read it // from tensorflow, i.e., adding a params. @@ -87,39 +105,32 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { // So we don't even need to do handle the epsilon for quantized kernel case. const float epsilon = 1e-6f; if (output->type == kTfLiteFloat32) { -#define TF_LITE_L2NORM(type) \ - tflite::L2NormalizationParams op_params; \ - op_params.input_zero_point = 0; \ - type::L2Normalization(op_params, GetTensorShape(input), \ - GetTensorData(input), GetTensorShape(output), \ - GetTensorData(output), epsilon) - - TF_LITE_L2NORM(reference_ops); -#undef TF_LITE_L2NORM + reference_ops::L2Normalization(data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), + epsilon); } else if (output->type == kTfLiteUInt8) { -#define TF_LITE_L2NORM(type) \ - tflite::L2NormalizationParams op_params; \ - op_params.input_zero_point = input->params.zero_point; \ - type::L2Normalization(op_params, GetTensorShape(input), \ - GetTensorData(input), GetTensorShape(output), \ - GetTensorData(output)) - - TF_LITE_L2NORM(reference_ops); -#undef TF_LITE_L2NORM + reference_ops::L2Normalization( + data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else if (output->type == kTfLiteInt8) { - const auto input_shape = GetTensorShape(input); - const auto output_shape = GetTensorShape(output); + const auto input_shape = tflite::micro::GetTensorShape(input); + const auto output_shape = tflite::micro::GetTensorShape(output); const int trailing_dim = input_shape.DimensionsCount() - 1; const int depth = MatchingDim(input_shape, trailing_dim, output_shape, trailing_dim); const int outer_size = MatchingFlatSizeSkipDim(input_shape, trailing_dim, output_shape); - reference_integer_ops::L2Normalization(input->params.zero_point, outer_size, - depth, GetTensorData(input), - GetTensorData(output)); + reference_integer_ops::L2Normalization( + data.input_zero_point, outer_size, depth, + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_KERNEL_LOG(context, "Output type is %d, requires float.", - output->type); + TF_LITE_KERNEL_LOG(context, "Output type is %s, requires float.", + TfLiteTypeGetName(output->type)); return kTfLiteError; } @@ -128,22 +139,18 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace l2norm -TfLiteRegistration* Register_L2NORM_REF() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/l2norm::Prepare, - /*invoke=*/l2norm::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - - return &r; +TfLiteRegistration Register_L2NORM_REF() { + return {/*init=*/l2norm::Init, + /*free=*/nullptr, + /*prepare=*/l2norm::Prepare, + /*invoke=*/l2norm::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_L2_NORMALIZATION() { - return Register_L2NORM_REF(); -} +TfLiteRegistration Register_L2_NORMALIZATION() { return Register_L2NORM_REF(); } } // namespace micro } // namespace ops diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/logical.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/logical.cc index c6a6a5a0..f4033ba8 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/logical.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/logical.cc @@ -15,8 +15,8 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/reference/binary_function.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" -#include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -31,20 +31,29 @@ constexpr int kOutputTensor = 0; TfLiteStatus LogicalImpl(TfLiteContext* context, TfLiteNode* node, bool (*func)(bool, bool)) { - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); - if (HaveSameShapes(input1, input2)) { + if (tflite::micro::HaveSameShapes(input1, input2)) { reference_ops::BinaryFunction( - GetTensorShape(input1), GetTensorData(input1), - GetTensorShape(input2), GetTensorData(input2), - GetTensorShape(output), GetTensorData(output), func); + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), func); } else { reference_ops::BroadcastBinaryFunction4DSlow( - GetTensorShape(input1), GetTensorData(input1), - GetTensorShape(input2), GetTensorData(input2), - GetTensorShape(output), GetTensorData(output), func); + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), func); } return kTfLiteOk; @@ -65,32 +74,30 @@ TfLiteStatus LogicalAndEval(TfLiteContext* context, TfLiteNode* node) { } // namespace } // namespace logical -TfLiteRegistration* Register_LOGICAL_OR() { +TfLiteRegistration Register_LOGICAL_OR() { // Init, Free, Prepare, Eval are satisfying the Interface required by // TfLiteRegistration. - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/logical::LogicalOrEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/logical::LogicalOrEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_LOGICAL_AND() { +TfLiteRegistration Register_LOGICAL_AND() { // Init, Free, Prepare, Eval are satisfying the Interface required by // TfLiteRegistration. - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/logical::LogicalAndEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/logical::LogicalAndEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/logistic.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/logistic.cc index cc360c58..3fa81ba8 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/logistic.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/logistic.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -42,9 +43,11 @@ struct OpData { TfLiteStatus CalculateArithmeticOpData(TfLiteContext* context, TfLiteNode* node, OpData* data) { const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); - TF_LITE_ENSURE_EQ(context, input->type, output->type); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); if (input->type == kTfLiteInt8) { TF_LITE_ENSURE_EQ(context, output->params.zero_point, std::numeric_limits::min()); @@ -54,6 +57,8 @@ TfLiteStatus CalculateArithmeticOpData(TfLiteContext* context, TfLiteNode* node, static_cast(input->params.scale) * static_cast(1 << (31 - kInputIntegerBits)); + data->input_zero_point = input->params.zero_point; + const double q = std::frexp(input_real_multiplier, &data->input_left_shift); data->input_multiplier = static_cast(TfLiteRound(q * (1ll << 31))); @@ -64,18 +69,34 @@ TfLiteStatus CalculateArithmeticOpData(TfLiteContext* context, TfLiteNode* node, } } // namespace +void* LogisticInit(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + +TfLiteStatus LogisticPrepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + + return CalculateArithmeticOpData(context, node, data); +} + TfLiteStatus LogisticEval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - OpData data; - CalculateArithmeticOpData(context, node, &data); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); if (input->type == kTfLiteFloat32) { switch (output->type) { case kTfLiteFloat32: { - reference_ops::Logistic( - GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + reference_ops::Logistic(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } default: @@ -88,10 +109,11 @@ TfLiteStatus LogisticEval(TfLiteContext* context, TfLiteNode* node) { switch (output->type) { case kTfLiteInt8: { reference_integer_ops::Logistic( - input->params.zero_point, data.input_range_radius, - data.input_multiplier, data.input_left_shift, - NumElements(input->dims), GetTensorData(input), - GetTensorData(output)); + data->input_zero_point, data->input_range_radius, + data->input_multiplier, data->input_left_shift, + NumElements(input->dims), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } default: @@ -113,16 +135,15 @@ TfLiteStatus LogisticEval(TfLiteContext* context, TfLiteNode* node) { } // namespace activations -TfLiteRegistration* Register_LOGISTIC() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/activations::LogisticEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_LOGISTIC() { + return {/*init=*/activations::LogisticInit, + /*free=*/nullptr, + /*prepare=*/activations::LogisticPrepare, + /*invoke=*/activations::LogisticEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro } // namespace ops diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/maximum_minimum.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/maximum_minimum.cc index 7162664c..a7c343bf 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/maximum_minimum.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/maximum_minimum.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -40,13 +41,13 @@ constexpr int kOutputTensor = 0; struct OpContext { OpContext(TfLiteContext* context, TfLiteNode* node) { - input1 = GetInput(context, node, kInputTensor1); - input2 = GetInput(context, node, kInputTensor2); - output = GetOutput(context, node, kOutputTensor); + input1 = tflite::micro::GetEvalInput(context, node, kInputTensor1); + input2 = tflite::micro::GetEvalInput(context, node, kInputTensor2); + output = tflite::micro::GetEvalOutput(context, node, kOutputTensor); } - const TfLiteTensor* input1; - const TfLiteTensor* input2; - TfLiteTensor* output; + const TfLiteEvalTensor* input1; + const TfLiteEvalTensor* input2; + TfLiteEvalTensor* output; }; struct MaximumOp { @@ -69,12 +70,12 @@ template void TFLiteOperation(TfLiteContext* context, TfLiteNode* node, const OpContext& op_context) { reference_ops::MaximumMinimumBroadcastSlow( - GetTensorShape(op_context.input1), - GetTensorData(op_context.input1), - GetTensorShape(op_context.input2), - GetTensorData(op_context.input2), - GetTensorShape(op_context.output), - GetTensorData(op_context.output), + tflite::micro::GetTensorShape(op_context.input1), + tflite::micro::GetTensorData(op_context.input1), + tflite::micro::GetTensorShape(op_context.input2), + tflite::micro::GetTensorData(op_context.input2), + tflite::micro::GetTensorShape(op_context.output), + tflite::micro::GetTensorData(op_context.output), op_type::template op); } @@ -116,34 +117,30 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace maximum_minimum -TfLiteRegistration* Register_MAXIMUM() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/ - maximum_minimum::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_MAXIMUM() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/ + maximum_minimum::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -TfLiteRegistration* Register_MINIMUM() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/ - maximum_minimum::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_MINIMUM() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/ + maximum_minimum::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/micro_ops.h b/code/lib/tfmicro/tensorflow/lite/micro/kernels/micro_ops.h index 83ccccaa..a65fc4f6 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/micro_ops.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/micro_ops.h @@ -17,10 +17,6 @@ limitations under the License. #include "tensorflow/lite/c/common.h" -namespace tflite { -namespace ops { -namespace micro { - // Forward declaration of all micro op kernel registration methods. These // registrations are included with the standard `BuiltinOpResolver`. // @@ -29,58 +25,73 @@ namespace micro { // their model requires, using a custom `(Micro)MutableOpResolver`. Selective // registration in turn allows the linker to strip unused kernels. -TfLiteRegistration* Register_ABS(); -TfLiteRegistration* Register_ADD(); -TfLiteRegistration* Register_ARG_MAX(); -TfLiteRegistration* Register_ARG_MIN(); -TfLiteRegistration* Register_AVERAGE_POOL_2D(); -TfLiteRegistration* Register_CEIL(); +namespace tflite { + +// TFLM is incrementally moving towards a flat tflite namespace +// (https://abseil.io/tips/130). Any new ops (or cleanup of existing ops should +// have their Register function declarations in the tflite namespace. + +TfLiteRegistration Register_CONV_2D(); +TfLiteRegistration Register_DEPTHWISE_CONV_2D(); +TfLiteRegistration Register_QUANTIZE(); +TfLiteRegistration Register_SHAPE(); +TfLiteRegistration Register_SOFTMAX(); +TfLiteRegistration Register_SVDF(); + +namespace ops { +namespace micro { + +TfLiteRegistration Register_ABS(); +TfLiteRegistration Register_ADD(); +TfLiteRegistration Register_ARG_MAX(); +TfLiteRegistration Register_ARG_MIN(); +TfLiteRegistration Register_AVERAGE_POOL_2D(); +TfLiteRegistration Register_CEIL(); +// TODO(b/160234179): Change custom OPs to also return by value. TfLiteRegistration* Register_CIRCULAR_BUFFER(); -TfLiteRegistration* Register_CONV_2D(); -TfLiteRegistration* Register_CONCATENATION(); -TfLiteRegistration* Register_COS(); -TfLiteRegistration* Register_DEPTHWISE_CONV_2D(); -TfLiteRegistration* Register_DEQUANTIZE(); -TfLiteRegistration* Register_EQUAL(); -TfLiteRegistration* Register_FLOOR(); -TfLiteRegistration* Register_FULLY_CONNECTED(); -TfLiteRegistration* Register_GREATER(); -TfLiteRegistration* Register_GREATER_EQUAL(); -TfLiteRegistration* Register_LESS(); -TfLiteRegistration* Register_LESS_EQUAL(); -TfLiteRegistration* Register_LOG(); -TfLiteRegistration* Register_LOGICAL_AND(); -TfLiteRegistration* Register_LOGICAL_NOT(); -TfLiteRegistration* Register_LOGICAL_OR(); -TfLiteRegistration* Register_LOGISTIC(); -TfLiteRegistration* Register_MAXIMUM(); -TfLiteRegistration* Register_MAX_POOL_2D(); -TfLiteRegistration* Register_MEAN(); -TfLiteRegistration* Register_MINIMUM(); -TfLiteRegistration* Register_MUL(); -TfLiteRegistration* Register_NEG(); -TfLiteRegistration* Register_NOT_EQUAL(); -TfLiteRegistration* Register_PACK(); -TfLiteRegistration* Register_PAD(); -TfLiteRegistration* Register_PADV2(); -TfLiteRegistration* Register_PRELU(); -TfLiteRegistration* Register_QUANTIZE(); -TfLiteRegistration* Register_RELU(); -TfLiteRegistration* Register_RELU6(); -TfLiteRegistration* Register_RESHAPE(); -TfLiteRegistration* Register_RESIZE_NEAREST_NEIGHBOR(); -TfLiteRegistration* Register_ROUND(); -TfLiteRegistration* Register_RSQRT(); -TfLiteRegistration* Register_SIN(); -TfLiteRegistration* Register_SOFTMAX(); -TfLiteRegistration* Register_SPLIT(); -TfLiteRegistration* Register_SQRT(); -TfLiteRegistration* Register_SQUARE(); -TfLiteRegistration* Register_STRIDED_SLICE(); -TfLiteRegistration* Register_SUB(); -TfLiteRegistration* Register_SVDF(); -TfLiteRegistration* Register_UNPACK(); -TfLiteRegistration* Register_L2_NORMALIZATION(); +TfLiteRegistration Register_CONCATENATION(); +TfLiteRegistration Register_COS(); +TfLiteRegistration Register_DEQUANTIZE(); +TfLiteRegistration Register_EQUAL(); +TfLiteRegistration Register_FLOOR(); +TfLiteRegistration Register_GREATER(); +TfLiteRegistration Register_GREATER_EQUAL(); +TfLiteRegistration Register_HARD_SWISH(); +TfLiteRegistration Register_LESS(); +TfLiteRegistration Register_LESS_EQUAL(); +TfLiteRegistration Register_LOG(); +TfLiteRegistration Register_LOGICAL_AND(); +TfLiteRegistration Register_LOGICAL_NOT(); +TfLiteRegistration Register_LOGICAL_OR(); +TfLiteRegistration Register_LOGISTIC(); +TfLiteRegistration Register_MAXIMUM(); +TfLiteRegistration Register_MAX_POOL_2D(); +TfLiteRegistration Register_MEAN(); +TfLiteRegistration Register_MINIMUM(); +TfLiteRegistration Register_MUL(); +TfLiteRegistration Register_NEG(); +TfLiteRegistration Register_NOT_EQUAL(); +TfLiteRegistration Register_PACK(); +TfLiteRegistration Register_PAD(); +TfLiteRegistration Register_PADV2(); +TfLiteRegistration Register_PRELU(); +TfLiteRegistration Register_REDUCE_MAX(); +TfLiteRegistration Register_RELU(); +TfLiteRegistration Register_RELU6(); +TfLiteRegistration Register_RESHAPE(); +TfLiteRegistration Register_RESIZE_NEAREST_NEIGHBOR(); +TfLiteRegistration Register_ROUND(); +TfLiteRegistration Register_RSQRT(); +TfLiteRegistration Register_SIN(); +TfLiteRegistration Register_SPLIT(); +TfLiteRegistration Register_SPLIT_V(); +TfLiteRegistration Register_SQRT(); +TfLiteRegistration Register_SQUARE(); +TfLiteRegistration Register_STRIDED_SLICE(); +TfLiteRegistration Register_SUB(); +TfLiteRegistration Register_UNPACK(); +TfLiteRegistration Register_L2_NORMALIZATION(); +TfLiteRegistration Register_TANH(); } // namespace micro } // namespace ops diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/mul.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/mul.cc index e89b58a3..b3f3bd4f 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/mul.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/mul.cc @@ -21,132 +21,194 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/memory_helpers.h" namespace tflite { namespace ops { namespace micro { namespace mul { +namespace { constexpr int kInput1Tensor = 0; constexpr int kInput2Tensor = 1; constexpr int kOutputTensor = 0; struct OpData { + int32_t input1_zero_point; + int32_t input2_zero_point; + int32_t output_activation_min; int32_t output_activation_max; - + int32_t output_zero_point; int32_t output_multiplier; int output_shift; + + float output_activation_min_f32; + float output_activation_max_f32; }; TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, TfLiteMulParams* params, OpData* data) { const TfLiteTensor* input1 = GetInput(context, node, kInput1Tensor); + TF_LITE_ENSURE(context, input1 != nullptr); const TfLiteTensor* input2 = GetInput(context, node, kInput2Tensor); + TF_LITE_ENSURE(context, input2 != nullptr); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); - TF_LITE_ENSURE_EQ(context, input1->type, input2->type); - - TF_LITE_ENSURE_STATUS(CalculateActivationRangeQuantized( - context, params->activation, output, &data->output_activation_min, - &data->output_activation_max)); + TF_LITE_ENSURE_TYPES_EQ(context, input1->type, input2->type); if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8) { + TF_LITE_ENSURE_STATUS(CalculateActivationRangeQuantized( + context, params->activation, output, &data->output_activation_min, + &data->output_activation_max)); + double real_multiplier = static_cast(input1->params.scale) * static_cast(input2->params.scale) / static_cast(output->params.scale); QuantizeMultiplier(real_multiplier, &data->output_multiplier, &data->output_shift); + + data->input1_zero_point = input1->params.zero_point; + data->input2_zero_point = input2->params.zero_point; + data->output_zero_point = output->params.zero_point; + } else { + CalculateActivationRange(params->activation, + &data->output_activation_min_f32, + &data->output_activation_max_f32); } return kTfLiteOk; } -void EvalQuantized(TfLiteContext* context, TfLiteNode* node, - TfLiteMulParams* params, OpData* data, - const TfLiteTensor* input1, const TfLiteTensor* input2, - TfLiteTensor* output) { - if (output->type == kTfLiteInt8 || output->type == kTfLiteUInt8) { - tflite::ArithmeticParams op_params; - SetActivationParams(data->output_activation_min, - data->output_activation_max, &op_params); - op_params.input1_offset = -input1->params.zero_point; - op_params.input2_offset = -input2->params.zero_point; - op_params.output_offset = output->params.zero_point; - op_params.output_multiplier = data->output_multiplier; - op_params.output_shift = data->output_shift; - bool need_broadcast = reference_ops::ProcessBroadcastShapes( - GetTensorShape(input1), GetTensorShape(input2), &op_params); +} // namespace -#define TF_LITE_MUL(type, opname, dtype) \ - type::opname(op_params, GetTensorShape(input1), \ - GetTensorData(input1), GetTensorShape(input2), \ - GetTensorData(input2), GetTensorShape(output), \ - GetTensorData(output)); +void EvalQuantized(TfLiteContext* context, TfLiteNode* node, const OpData* data, + const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, TfLiteEvalTensor* output) { + tflite::ArithmeticParams op_params = {}; + op_params.quantized_activation_min = data->output_activation_min; + op_params.quantized_activation_max = data->output_activation_max; + op_params.float_activation_max = data->output_activation_max_f32; + op_params.input1_offset = -data->input1_zero_point; + op_params.input2_offset = -data->input2_zero_point; + op_params.output_offset = data->output_zero_point; + op_params.output_multiplier = data->output_multiplier; + op_params.output_shift = data->output_shift; - if (output->type == kTfLiteInt8) { - if (need_broadcast) { - TF_LITE_MUL(reference_integer_ops, BroadcastMul4DSlow, int8_t); - } else { - TF_LITE_MUL(reference_integer_ops, Mul, int8_t); - } - } else if (output->type == kTfLiteUInt8) { - if (need_broadcast) { - TF_LITE_MUL(reference_ops, BroadcastMul4DSlow, uint8_t); - } else { - TF_LITE_MUL(reference_ops, Mul, uint8_t); - } + bool need_broadcast = reference_ops::ProcessBroadcastShapes( + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorShape(input2), &op_params); + + if (output->type == kTfLiteInt8) { + if (need_broadcast) { + reference_integer_ops::BroadcastMul4DSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } else { + reference_integer_ops::Mul(op_params, + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } + } else if (output->type == kTfLiteUInt8) { + if (need_broadcast) { + reference_integer_ops::BroadcastMul4DSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } else { + reference_integer_ops::Mul(op_params, + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } -#undef TF_LITE_MUL } } void EvalFloat(TfLiteContext* context, TfLiteNode* node, - TfLiteMulParams* params, OpData* data, - const TfLiteTensor* input1, const TfLiteTensor* input2, - TfLiteTensor* output) { - float output_activation_min, output_activation_max; - CalculateActivationRange(params->activation, &output_activation_min, - &output_activation_max); - tflite::ArithmeticParams op_params; - SetActivationParams(output_activation_min, output_activation_max, &op_params); + TfLiteMulParams* params, const OpData* data, + const TfLiteEvalTensor* input1, const TfLiteEvalTensor* input2, + TfLiteEvalTensor* output) { + tflite::ArithmeticParams op_params = {}; + op_params.float_activation_min = data->output_activation_min_f32; + op_params.float_activation_max = data->output_activation_max_f32; bool need_broadcast = reference_ops::ProcessBroadcastShapes( - GetTensorShape(input1), GetTensorShape(input2), &op_params); -#define TF_LITE_MUL(opname) \ - reference_ops::opname(op_params, GetTensorShape(input1), \ - GetTensorData(input1), GetTensorShape(input2), \ - GetTensorData(input2), GetTensorShape(output), \ - GetTensorData(output)); + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorShape(input2), &op_params); if (need_broadcast) { - TF_LITE_MUL(BroadcastMul4DSlow); + reference_ops::BroadcastMul4DSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_MUL(Mul); + reference_ops::Mul(op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } -#undef TF_LITE_MUL +} + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->builtin_data != nullptr); + auto* params = reinterpret_cast(node->builtin_data); + + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + + return CalculateOpData(context, node, params, data); } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->builtin_data != nullptr); auto* params = reinterpret_cast(node->builtin_data); - OpData data; - const TfLiteTensor* input1 = GetInput(context, node, kInput1Tensor); - const TfLiteTensor* input2 = GetInput(context, node, kInput2Tensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); - CalculateOpData(context, node, params, &data); + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInput1Tensor); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInput2Tensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); switch (input1->type) { case kTfLiteUInt8: case kTfLiteInt8: - EvalQuantized(context, node, params, &data, input1, input2, output); + EvalQuantized(context, node, data, input1, input2, output); break; case kTfLiteFloat32: - EvalFloat(context, node, params, &data, input1, input2, output); + EvalFloat(context, node, params, data, input1, input2, output); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -158,16 +220,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } } // namespace mul -TfLiteRegistration* Register_MUL() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/mul::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_MUL() { + return {/*init=*/mul::Init, + /*free=*/nullptr, + /*prepare=*/mul::Prepare, + /*invoke=*/mul::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/neg.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/neg.cc index 570215a0..74a95ca3 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/neg.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/neg.cc @@ -17,7 +17,7 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" -#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -28,14 +28,17 @@ constexpr int kInputTensor = 0; constexpr int kOutputTensor = 0; TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); switch (input->type) { // TODO(wangtz): handle for kTfLiteInt8 case kTfLiteFloat32: - reference_ops::Negate(GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), - GetTensorData(output)); + reference_ops::Negate(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", @@ -47,16 +50,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace neg -TfLiteRegistration* Register_NEG() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/neg::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_NEG() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/neg::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/pack.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/pack.cc index 60a23cc0..d332fc63 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/pack.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/pack.cc @@ -16,7 +16,7 @@ limitations under the License. #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" -#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -28,9 +28,11 @@ constexpr int kOutputTensor = 0; template TfLiteStatus PackImpl(TfLiteContext* context, TfLiteNode* node, - TfLiteTensor* output, int values_count, int axis) { + TfLiteEvalTensor* output, int values_count, int axis) { + const TfLiteEvalTensor* input0 = + tflite::micro::GetEvalInput(context, node, 0); + const int dimensions = output->dims->size; - const TfLiteTensor* input0 = GetInput(context, node, 0); const TfLiteIntArray* input_dims = input0->dims; const TfLiteIntArray* output_dims = output->dims; @@ -52,11 +54,11 @@ TfLiteStatus PackImpl(TfLiteContext* context, TfLiteNode* node, } TFLITE_DCHECK_EQ(input_size, copy_size * outer_size); - T* output_data = GetTensorData(output); + T* output_data = tflite::micro::GetTensorData(output); for (int i = 0; i < values_count; ++i) { - const TfLiteTensor* t = GetInput(context, node, i); - const T* input_data = GetTensorData(t); + const TfLiteEvalTensor* t = tflite::micro::GetEvalInput(context, node, i); + const T* input_data = tflite::micro::GetTensorData(t); for (int k = 0; k < outer_size; ++k) { const T* input_ptr = input_data + copy_size * k; int loc = k * values_count * copy_size + i * copy_size; @@ -72,7 +74,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLitePackParams* data = reinterpret_cast(node->builtin_data); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); switch (output->type) { case kTfLiteFloat32: { @@ -108,16 +111,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace } // namespace pack -TfLiteRegistration* Register_PACK() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/pack::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_PACK() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/pack::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/pad.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/pad.cc index 20e68802..5d9d4364 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/pad.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/pad.cc @@ -16,189 +16,208 @@ limitations under the License. #include -#include "tensorflow/lite/kernels/internal/types.h" - -#ifdef MEMORY_SANITIZER -#include -#else -#define __msan_check_mem_is_initialized(ptr, size) -#endif - #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/kernels/internal/tensor.h" +#include "tensorflow/lite/kernels/internal/portable_tensor.h" +#include "tensorflow/lite/kernels/internal/types.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { namespace micro { namespace pad { +namespace { -struct PadContext { - PadContext(TfLiteContext* context, TfLiteNode* node) { - input = GetInput(context, node, 0); - paddings = GetInput(context, node, 1); - constant_values = nullptr; - if (NumInputs(node) == 3) { - constant_values = GetOptionalInputTensor(context, node, 2); - } else { - constant_values = nullptr; - } - output = GetOutput(context, node, 0); - dims = NumDimensions(input); - - resizing_category = ResizingCategory::kGenericResize; - const int paddings_total = GetTensorShape(paddings).FlatSize(); - const int32* paddings_data = GetTensorData(paddings); - // Paddings will be a n,2 array, and we need to detect 4D arrays with the - // pattern { {0,0}, {a, b}, {c, d}, {0,0} }. - if (IsConstantTensor(paddings) && paddings_total == 8 && - (paddings_data[0] == 0 && paddings_data[1] == 0) && - (paddings_data[6] == 0 && paddings_data[7] == 0)) { - resizing_category = ResizingCategory::kImageStyle; - } - } - const TfLiteTensor* constant_values; - const TfLiteTensor* input; - const TfLiteTensor* paddings; - TfLiteTensor* output; - int dims; - ResizingCategory resizing_category; +struct OpData { + PadParams params; + int32_t output_zero_point; }; +} // namespace + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + TF_LITE_ENSURE(context, NumInputs(node) == 2 || NumInputs(node) == 3); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); - PadContext op_context(context, node); - TF_LITE_ENSURE_EQ(context, op_context.input->type, op_context.output->type); - if (op_context.constant_values != nullptr) { - TF_LITE_ENSURE_EQ(context, op_context.input->type, - op_context.constant_values->type); + const TfLiteTensor* input = GetInput(context, node, /*index=*/0); + TF_LITE_ENSURE(context, input != nullptr); + const TfLiteTensor* paddings = GetInput(context, node, /*index=*/1); + TF_LITE_ENSURE(context, paddings != nullptr); + const TfLiteTensor* constant_values = + NumInputs(node) == 3 ? GetInput(context, node, /*index=*/2) : nullptr; + TfLiteTensor* output = GetOutput(context, node, /*index=*/0); + TF_LITE_ENSURE(context, output != nullptr); + + TF_LITE_ENSURE_EQ(context, input->type, output->type); + + // Current implementations rely on the inputs being <= 4D. + TF_LITE_ENSURE(context, NumDimensions(input) <= + reference_ops::PadKernelMaxDimensionCount()); + + if (constant_values != nullptr) { + TF_LITE_ENSURE_EQ(context, input->type, constant_values->type); + // Ensure that constant_values is a scalar. + TF_LITE_ENSURE_EQ(context, NumElements(constant_values), 1); } // There must be a pair of paddings for each output dimension. - TF_LITE_ENSURE_EQ(context, GetTensorShape(op_context.paddings).FlatSize(), - op_context.output->dims->size * 2); + TF_LITE_ENSURE_EQ(context, GetTensorShape(paddings).FlatSize(), + output->dims->size * 2); // On Micro, outputs must be properly sized by the converter. - const int32* paddings_data = GetTensorData(op_context.paddings); - for (int i = 0; i < op_context.output->dims->size; i++) { - int output_dim = op_context.output->dims->data[i]; - int expected_dim = op_context.input->dims->data[i] + paddings_data[i * 2] + - paddings_data[i * 2 + 1]; + // NOTE: This data is only available because the paddings buffer is stored in + // the flatbuffer: + TF_LITE_ENSURE(context, IsConstantTensor(paddings)); + const int32_t* paddings_data = GetTensorData(paddings); + for (int i = 0; i < output->dims->size; i++) { + int output_dim = output->dims->data[i]; + int expected_dim = + input->dims->data[i] + paddings_data[i * 2] + paddings_data[i * 2 + 1]; TF_LITE_ENSURE_EQ(context, output_dim, expected_dim); } - // Current implementations rely on the inputs being <= 4D. - TF_LITE_ENSURE( - context, op_context.dims <= reference_ops::PadKernelMaxDimensionCount()); - TF_LITE_ENSURE(context, IsConstantTensor(op_context.paddings)); + // Calculate OpData: + data->params.resizing_category = ResizingCategory::kGenericResize; + const int paddings_total = GetTensorShape(paddings).FlatSize(); + if (paddings_total == 8 && (paddings_data[0] == 0 && paddings_data[1] == 0) && + (paddings_data[6] == 0 && paddings_data[7] == 0)) { + data->params.resizing_category = ResizingCategory::kImageStyle; + } + + const int num_input_dimensions = NumDimensions(input); + data->params.left_padding_count = num_input_dimensions; + data->params.right_padding_count = num_input_dimensions; + + for (int idx = num_input_dimensions - 1; idx >= 0; --idx) { + data->params.left_padding[idx] = paddings_data[idx * 2]; + data->params.right_padding[idx] = paddings_data[idx * 2 + 1]; + } + + if (input->type == kTfLiteInt8 || input->type == kTfLiteUInt8) { + if (constant_values == nullptr) { + // Quantized Pad requires that 0 is represented in the quantized + // range. + if (input->type == kTfLiteUInt8) { + TF_LITE_ENSURE(context, output->params.zero_point >= + std::numeric_limits::min()); + TF_LITE_ENSURE(context, output->params.zero_point <= + std::numeric_limits::max()); + } else { + TF_LITE_ENSURE(context, output->params.zero_point >= + std::numeric_limits::min()); + TF_LITE_ENSURE(context, output->params.zero_point <= + std::numeric_limits::max()); + } + } else { + // Quantized Pad requires that 'constant_values' is represented in the + // same quantized range as the input and output tensors. + TF_LITE_ENSURE_EQ(context, output->params.zero_point, + constant_values->params.zero_point); + TF_LITE_ENSURE_EQ(context, static_cast(output->params.scale), + static_cast(constant_values->params.scale)); + } + data->output_zero_point = output->params.zero_point; + } + return kTfLiteOk; } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - PadContext op_context(context, node); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); - if (op_context.constant_values != nullptr) { - // Ensure that constant_values is a scalar. - TF_LITE_ENSURE_EQ(context, NumElements(op_context.constant_values), 1); - } + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, /*index=*/0); + const TfLiteEvalTensor* constant_values = + NumInputs(node) == 3 + ? tflite::micro::GetEvalInput(context, node, /*index=*/2) + : nullptr; + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, /*index=*/0); - // Create before and after padding arrays that are accepted by the kernel. - const int32* paddings_data = GetTensorData(op_context.paddings); - - tflite::PadParams op_params; - memset(&op_params, 0, sizeof(PadParams)); - op_params.left_padding_count = op_context.dims; - op_params.right_padding_count = op_context.dims; - - for (int idx = op_context.dims - 1; idx >= 0; --idx) { - op_params.left_padding[idx] = paddings_data[idx * 2]; - op_params.right_padding[idx] = paddings_data[idx * 2 + 1]; - } - -#define TF_LITE_PAD(type, op_name, scalar, pad_value) \ - const scalar pad_value_copy = pad_value; \ - \ - type::op_name(op_params, GetTensorShape(op_context.input), \ - GetTensorData(op_context.input), &pad_value_copy, \ - GetTensorShape(op_context.output), \ - GetTensorData(op_context.output)) - switch (op_context.input->type) { + switch (input->type) { case kTfLiteFloat32: { - float pad_value = op_context.constant_values == nullptr - ? 0.f - : *GetTensorData(op_context.constant_values); - if (op_context.resizing_category == ResizingCategory::kImageStyle) { - TF_LITE_PAD(reference_ops, PadImageStyle, float, pad_value); + float pad_value = + constant_values == nullptr + ? 0.f + : *tflite::micro::GetTensorData(constant_values); + if (data->params.resizing_category == ResizingCategory::kImageStyle) { + reference_ops::PadImageStyle( + data->params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), &pad_value, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_PAD(reference_ops, Pad, float, pad_value); + reference_ops::Pad(data->params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + &pad_value, tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } break; case kTfLiteUInt8: { uint8_t pad_value; - if (op_context.constant_values == nullptr) { - // Quantized Pad requires that 0 is represented in the quantized - // range. - TF_LITE_ENSURE(context, op_context.output->params.zero_point >= - std::numeric_limits::min()); - TF_LITE_ENSURE(context, op_context.output->params.zero_point <= - std::numeric_limits::max()); - pad_value = static_cast(op_context.output->params.zero_point); + if (constant_values == nullptr) { + pad_value = static_cast(data->output_zero_point); } else { - // Quantized Pad requires that 'constant_values' is represented in the - // same quantized range as the input and output tensors. - TF_LITE_ENSURE_EQ(context, op_context.output->params.zero_point, - op_context.constant_values->params.zero_point); - TF_LITE_ENSURE_EQ( - context, static_cast(op_context.output->params.scale), - static_cast(op_context.constant_values->params.scale)); - pad_value = *GetTensorData(op_context.constant_values); + pad_value = *tflite::micro::GetTensorData(constant_values); } - if (op_context.resizing_category == ResizingCategory::kImageStyle) { - TF_LITE_PAD(reference_ops, PadImageStyle, uint8_t, pad_value); + if (data->params.resizing_category == ResizingCategory::kImageStyle) { + reference_ops::PadImageStyle( + data->params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), &pad_value, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_PAD(reference_ops, Pad, uint8_t, pad_value); + reference_ops::Pad(data->params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + &pad_value, tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } break; case kTfLiteInt8: { int8_t pad_value; - if (op_context.constant_values == nullptr) { - // Quantized Pad requires that 0 is represented in the quantized - // range. - TF_LITE_ENSURE(context, op_context.output->params.zero_point >= - std::numeric_limits::min()); - TF_LITE_ENSURE(context, op_context.output->params.zero_point <= - std::numeric_limits::max()); - pad_value = static_cast(op_context.output->params.zero_point); + if (constant_values == nullptr) { + pad_value = static_cast(data->output_zero_point); } else { - // Quantized Pad requires that 'constant_values' is represented in the - // same quantized range as the input and output tensors. - TF_LITE_ENSURE_EQ(context, op_context.output->params.zero_point, - op_context.constant_values->params.zero_point); - TF_LITE_ENSURE(context, op_context.output->params.scale == - op_context.constant_values->params.scale); - pad_value = *GetTensorData(op_context.constant_values); + pad_value = *tflite::micro::GetTensorData(constant_values); } - if (op_context.resizing_category == ResizingCategory::kImageStyle) { - TF_LITE_PAD(reference_ops, PadImageStyle, int8_t, pad_value); + if (data->params.resizing_category == ResizingCategory::kImageStyle) { + reference_ops::PadImageStyle( + data->params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), &pad_value, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_PAD(reference_ops, Pad, int8_t, pad_value); + reference_ops::Pad(data->params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + &pad_value, tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } break; case kTfLiteInt32: { int32_t pad_value = - op_context.constant_values == nullptr + constant_values == nullptr ? 0 - : *GetTensorData(op_context.constant_values); - TF_LITE_PAD(reference_ops, Pad, int32_t, pad_value); + : *tflite::micro::GetTensorData(constant_values); + reference_ops::Pad(data->params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + &pad_value, tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } break; default: TF_LITE_KERNEL_LOG(context, "Type %s not currently supported by Pad.", - TfLiteTypeGetName(op_context.input->type)); + TfLiteTypeGetName(input->type)); return kTfLiteError; } #undef TF_LITE_PAD @@ -207,29 +226,27 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace pad -TfLiteRegistration* Register_PAD() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/pad::Prepare, - /*invoke=*/pad::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_PAD() { + return {/*init=*/pad::Init, + /*free=*/nullptr, + /*prepare=*/pad::Prepare, + /*invoke=*/pad::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } // Also register Pad as PadV2. -TfLiteRegistration* Register_PADV2() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/pad::Prepare, - /*invoke=*/pad::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_PADV2() { + return {/*init=*/pad::Init, + /*free=*/nullptr, + /*prepare=*/pad::Prepare, + /*invoke=*/pad::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/pooling.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/pooling.cc index 66c873fb..64aef0e1 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/pooling.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/pooling.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/padding.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -32,6 +33,10 @@ constexpr int kOutputTensor = 0; struct OpData { TfLitePaddingValues padding; + int32_t activation_min; + int32_t activation_max; + float activation_min_f32; + float activation_max_f32; }; TfLiteStatus CalculateOpData(const TfLiteContext* context, @@ -55,11 +60,7 @@ TfLiteStatus CalculateOpData(const TfLiteContext* context, void AverageEvalFloat(const TfLiteContext* context, const TfLiteNode* node, const TfLitePoolParams* params, const OpData* data, - const TfLiteTensor* input, TfLiteTensor* output) { - float activation_min, activation_max; - CalculateActivationRange(params->activation, &activation_min, - &activation_max); - + const TfLiteEvalTensor* input, TfLiteEvalTensor* output) { PoolParams op_params; op_params.stride_height = params->stride_height; op_params.stride_width = params->stride_width; @@ -67,20 +68,19 @@ void AverageEvalFloat(const TfLiteContext* context, const TfLiteNode* node, op_params.filter_width = params->filter_width; op_params.padding_values.height = data->padding.height; op_params.padding_values.width = data->padding.width; - op_params.float_activation_min = activation_min; - op_params.float_activation_max = activation_max; - reference_ops::AveragePool( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + op_params.float_activation_min = data->activation_min_f32; + op_params.float_activation_max = data->activation_max_f32; + reference_ops::AveragePool(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } void AverageEvalQuantized(TfLiteContext* context, const TfLiteNode* node, const TfLitePoolParams* params, const OpData* data, - const TfLiteTensor* input, TfLiteTensor* output) { + const TfLiteEvalTensor* input, + TfLiteEvalTensor* output) { TFLITE_DCHECK(input->type == kTfLiteUInt8 || input->type == kTfLiteInt8); - int32_t activation_min, activation_max; - (void)CalculateActivationRangeQuantized(context, params->activation, output, - &activation_min, &activation_max); PoolParams op_params; op_params.stride_height = params->stride_height; @@ -89,27 +89,26 @@ void AverageEvalQuantized(TfLiteContext* context, const TfLiteNode* node, op_params.filter_width = params->filter_width; op_params.padding_values.height = data->padding.height; op_params.padding_values.width = data->padding.width; - op_params.quantized_activation_min = activation_min; - op_params.quantized_activation_max = activation_max; + op_params.quantized_activation_min = data->activation_min; + op_params.quantized_activation_max = data->activation_max; if (input->type == kTfLiteUInt8) { - reference_ops::AveragePool( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + reference_ops::AveragePool(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { reference_integer_ops::AveragePool( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } void MaxEvalFloat(TfLiteContext* context, TfLiteNode* node, - TfLitePoolParams* params, OpData* data, - const TfLiteTensor* input, TfLiteTensor* output) { - float activation_min, activation_max; - CalculateActivationRange(params->activation, &activation_min, - &activation_max); - + TfLitePoolParams* params, const OpData* data, + const TfLiteEvalTensor* input, TfLiteEvalTensor* output) { tflite::PoolParams op_params; op_params.stride_height = params->stride_height; op_params.stride_width = params->stride_width; @@ -117,22 +116,17 @@ void MaxEvalFloat(TfLiteContext* context, TfLiteNode* node, op_params.filter_width = params->filter_width; op_params.padding_values.height = data->padding.height; op_params.padding_values.width = data->padding.width; - op_params.float_activation_min = activation_min; - op_params.float_activation_max = activation_max; - reference_ops::MaxPool(op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(output), - GetTensorData(output)); + op_params.float_activation_min = data->activation_min_f32; + op_params.float_activation_max = data->activation_max_f32; + reference_ops::MaxPool(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } void MaxEvalQuantized(TfLiteContext* context, TfLiteNode* node, - TfLitePoolParams* params, OpData* data, - const TfLiteTensor* input, TfLiteTensor* output) { - TFLITE_DCHECK(input->type == kTfLiteUInt8 || input->type == kTfLiteInt8); - - int32_t activation_min, activation_max; - (void)CalculateActivationRangeQuantized(context, params->activation, output, - &activation_min, &activation_max); - + TfLitePoolParams* params, const OpData* data, + const TfLiteEvalTensor* input, TfLiteEvalTensor* output) { tflite::PoolParams op_params; op_params.stride_height = params->stride_height; op_params.stride_width = params->stride_width; @@ -140,39 +134,44 @@ void MaxEvalQuantized(TfLiteContext* context, TfLiteNode* node, op_params.filter_width = params->filter_width; op_params.padding_values.height = data->padding.height; op_params.padding_values.width = data->padding.width; - op_params.quantized_activation_min = activation_min; - op_params.quantized_activation_max = activation_max; + op_params.quantized_activation_min = data->activation_min; + op_params.quantized_activation_max = data->activation_max; if (input->type == kTfLiteUInt8) { - reference_ops::MaxPool( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + reference_ops::MaxPool(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { reference_integer_ops::MaxPool( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } } // namespace - TfLiteStatus AverageEval(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->builtin_data != nullptr); auto* params = reinterpret_cast(node->builtin_data); - OpData data; - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); - TF_LITE_ENSURE_STATUS(CalculateOpData(context, params, input, output, &data)); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); // Inputs and outputs share the same type, guaranteed by the converter. switch (input->type) { case kTfLiteFloat32: - AverageEvalFloat(context, node, params, &data, input, output); + AverageEvalFloat(context, node, params, data, input, output); break; case kTfLiteUInt8: case kTfLiteInt8: - AverageEvalQuantized(context, node, params, &data, input, output); + AverageEvalQuantized(context, node, params, data, input, output); break; default: TF_LITE_KERNEL_LOG(context, "Input type %s is not currently supported", @@ -183,21 +182,24 @@ TfLiteStatus AverageEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus MaxEval(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->builtin_data != nullptr); auto* params = reinterpret_cast(node->builtin_data); - OpData data; - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData* data = static_cast(node->user_data); - TF_LITE_ENSURE_STATUS(CalculateOpData(context, params, input, output, &data)); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); switch (input->type) { case kTfLiteFloat32: - MaxEvalFloat(context, node, params, &data, input, output); + MaxEvalFloat(context, node, params, data, input, output); break; case kTfLiteUInt8: case kTfLiteInt8: - MaxEvalQuantized(context, node, params, &data, input, output); + MaxEvalQuantized(context, node, params, data, input, output); break; default: TF_LITE_KERNEL_LOG(context, "Type %s not currently supported.", @@ -207,30 +209,59 @@ TfLiteStatus MaxEval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } -} // namespace pooling - -TfLiteRegistration* Register_AVERAGE_POOL_2D() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/pooling::AverageEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); } -TfLiteRegistration* Register_MAX_POOL_2D() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/pooling::MaxEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->builtin_data != nullptr); + auto* params = reinterpret_cast(node->builtin_data); + + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + + const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + TF_LITE_ENSURE_STATUS(CalculateOpData(context, params, input, output, data)); + + if (input->type == kTfLiteFloat32) { + CalculateActivationRange(params->activation, &data->activation_min_f32, + &data->activation_max_f32); + } else if (input->type == kTfLiteInt8 || input->type == kTfLiteUInt8) { + CalculateActivationRangeQuantized(context, params->activation, output, + &data->activation_min, + &data->activation_max); + } + + return kTfLiteOk; +} + +} // namespace pooling + +TfLiteRegistration Register_AVERAGE_POOL_2D() { + return {/*init=*/pooling::Init, + /*free=*/nullptr, + /*prepare=*/pooling::Prepare, + /*invoke=*/pooling::AverageEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; +} + +TfLiteRegistration Register_MAX_POOL_2D() { + return {/*init=*/pooling::Init, + /*free=*/nullptr, + /*prepare=*/pooling::Prepare, + /*invoke=*/pooling::MaxEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/prelu.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/prelu.cc index a20d2c88..b48491d6 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/prelu.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/prelu.cc @@ -15,20 +15,45 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/reference/prelu.h" +#include + #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { namespace micro { namespace activations { +namespace { + +TfLiteStatus CalculatePreluParams(const TfLiteTensor* input, + const TfLiteTensor* alpha, + TfLiteTensor* output, PreluParams* params) { + if (output->type == kTfLiteInt8 || output->type == kTfLiteUInt8 || + output->type == kTfLiteInt16) { + double real_multiplier_1 = static_cast(input->params.scale) / + static_cast(output->params.scale); + double real_multiplier_2 = static_cast(input->params.scale) * + static_cast(alpha->params.scale) / + static_cast(output->params.scale); + QuantizeMultiplier(real_multiplier_1, ¶ms->output_multiplier_1, + ¶ms->output_shift_1); + QuantizeMultiplier(real_multiplier_2, ¶ms->output_multiplier_2, + ¶ms->output_shift_2); + + params->input_offset = -input->params.zero_point; + params->alpha_offset = -alpha->params.zero_point; + params->output_offset = output->params.zero_point; + } -TfLiteStatus PreluPrepare(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } +} // namespace + inline void BroadcastPrelu4DSlowFloat( const RuntimeShape& unextended_input1_shape, const float* input1_data, const RuntimeShape& unextended_input2_shape, const float* input2_data, @@ -60,43 +85,67 @@ inline void BroadcastPrelu4DSlowFloat( } } -TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) { +void* PreluInit(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(PreluParams)); +} + +TfLiteStatus PreluPrepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + PreluParams* params = static_cast(node->user_data); + const TfLiteTensor* input = GetInput(context, node, 0); + TF_LITE_ENSURE(context, input != nullptr); const TfLiteTensor* alpha = GetInput(context, node, 1); + TF_LITE_ENSURE(context, alpha != nullptr); TfLiteTensor* output = GetOutput(context, node, 0); - int32_t output_multiplier = 0; - int output_shift = 0; - if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt16) { - double real_multiplier = static_cast(input->params.scale) * - static_cast(alpha->params.scale) / - static_cast(output->params.scale); - QuantizeMultiplierSmallerThanOneExp(real_multiplier, &output_multiplier, - &output_shift); - } + TF_LITE_ENSURE(context, output != nullptr); + + return CalculatePreluParams(input, alpha, output, params); +} + +TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + const PreluParams& params = + *(static_cast(node->user_data)); + + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + const TfLiteEvalTensor* alpha = tflite::micro::GetEvalInput(context, node, 1); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); + switch (input->type) { case kTfLiteFloat32: { - BroadcastPrelu4DSlowFloat( - GetTensorShape(input), GetTensorData(input), - GetTensorShape(alpha), GetTensorData(alpha), - GetTensorShape(output), GetTensorData(output)); + BroadcastPrelu4DSlowFloat(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(alpha), + tflite::micro::GetTensorData(alpha), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } break; case kTfLiteUInt8: { - PreluParams op_params; - op_params.input_offset = -input->params.zero_point; - op_params.alpha_offset = -alpha->params.zero_point; - op_params.output_offset = output->params.zero_point; - op_params.output_multiplier = output_multiplier; - op_params.output_shift = output_shift; reference_ops::BroadcastPrelu4DSlow( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(alpha), GetTensorData(alpha), - GetTensorShape(output), GetTensorData(output)); + params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(alpha), + tflite::micro::GetTensorData(alpha), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + return kTfLiteOk; + } break; + case kTfLiteInt8: { + reference_ops::BroadcastPrelu4DSlow( + params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(alpha), + tflite::micro::GetTensorData(alpha), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } break; default: TF_LITE_KERNEL_LOG( - context, "Only float32 and uint8 are supported currently, got %d.", + context, "Only float32 and uint8_t are supported currently, got %d.", TfLiteTypeGetName(input->type)); return kTfLiteError; } @@ -104,16 +153,15 @@ TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) { } // namespace activations -TfLiteRegistration* Register_PRELU() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/activations::PreluPrepare, - /*invoke=*/activations::PreluEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_PRELU() { + return {/*init=*/activations::PreluInit, + /*free=*/nullptr, + /*prepare=*/activations::PreluPrepare, + /*invoke=*/activations::PreluEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/quantize.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/quantize.cc index d40471df..f6d8c927 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/quantize.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/quantize.cc @@ -19,19 +19,38 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/reference/requantize.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" #include "tensorflow/lite/micro/micro_utils.h" namespace tflite { -namespace ops { -namespace micro { -namespace quantize { +namespace { + +struct OpData { + tflite::QuantizationParams quantization_params; + // The scaling factor from input to output (aka the 'real multiplier') can + // be represented as a fixed point multiplier plus a left shift. + int32_t output_multiplier; + int output_shift; + + int32_t input_zero_point; +}; + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); const TfLiteTensor* input = GetInput(context, node, 0); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, 0); + TF_LITE_ENSURE(context, output != nullptr); // TODO(b/128934713): Add support for fixed-point per-channel quantization. // Currently this only support affine per-layer quantization. @@ -43,34 +62,61 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE(context, affine_quantization->scale); TF_LITE_ENSURE(context, affine_quantization->scale->size == 1); - TF_LITE_ENSURE(context, - input->type == kTfLiteFloat32 || input->type == kTfLiteInt16); - TF_LITE_ENSURE(context, - output->type == kTfLiteUInt8 || output->type == kTfLiteInt8); + TF_LITE_ENSURE(context, input->type == kTfLiteFloat32 || + input->type == kTfLiteInt16 || + input->type == kTfLiteInt8); + TF_LITE_ENSURE(context, output->type == kTfLiteUInt8 || + output->type == kTfLiteInt8 || + output->type == kTfLiteInt16 || + output->type == kTfLiteInt32); + if (((input->type == kTfLiteInt16 || input->type == kTfLiteInt8) && + output->type == kTfLiteInt8) || + (input->type == kTfLiteInt16 && output->type == kTfLiteInt16)) { + double effective_scale = static_cast(input->params.scale) / + static_cast(output->params.scale); + + QuantizeMultiplier(effective_scale, &data->output_multiplier, + &data->output_shift); + } + + data->quantization_params.zero_point = output->params.zero_point; + data->quantization_params.scale = static_cast(output->params.scale); + + data->input_zero_point = input->params.zero_point; return kTfLiteOk; } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, 0); - TfLiteTensor* output = GetOutput(context, node, 0); + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); - tflite::QuantizationParams op_params; - op_params.zero_point = output->params.zero_point; - op_params.scale = static_cast(output->params.scale); + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); if (input->type == kTfLiteFloat32) { switch (output->type) { case kTfLiteInt8: reference_ops::AffineQuantize( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + data->quantization_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; case kTfLiteUInt8: reference_ops::AffineQuantize( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + data->quantization_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; + case kTfLiteInt16: + reference_ops::AffineQuantize( + data->quantization_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + return kTfLiteOk; default: TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", TfLiteTypeGetName(input->type), @@ -79,17 +125,45 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } } else if (input->type == kTfLiteInt16) { size_t size = ElementCount(*input->dims); - int32_t output_multiplier; - int output_shift; - double effective_scale = - static_cast(input->params.scale / output->params.scale); switch (output->type) { case kTfLiteInt8: - QuantizeMultiplier(effective_scale, &output_multiplier, &output_shift); + reference_ops::Requantize(tflite::micro::GetTensorData(input), + size, data->output_multiplier, + data->output_shift, data->input_zero_point, + data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); + break; + case kTfLiteInt16: reference_ops::Requantize( - GetTensorData(input), size, output_multiplier, - output_shift, input->params.zero_point, output->params.zero_point, - GetTensorData(output)); + tflite::micro::GetTensorData(input), size, + data->output_multiplier, data->output_shift, data->input_zero_point, + data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); + return kTfLiteOk; + case kTfLiteInt32: + reference_ops::Requantize( + tflite::micro::GetTensorData(input), size, + data->output_multiplier, data->output_shift, data->input_zero_point, + data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); + return kTfLiteOk; + default: + TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", + TfLiteTypeGetName(input->type), + TfLiteTypeGetName(output->type)); + return kTfLiteError; + } + } else if (input->type == kTfLiteInt8) { + // Int8 to Int8 requantization, required if the input and output tensors + // have different scales and/or zero points. + size_t size = ElementCount(*input->dims); + switch (output->type) { + case kTfLiteInt8: + reference_ops::Requantize(tflite::micro::GetTensorData(input), + size, data->output_multiplier, + data->output_shift, data->input_zero_point, + data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); break; default: TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", @@ -107,23 +181,17 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } -} // namespace quantize +} // namespace -// This Op (QUANTIZE) quantizes the input and produces quantized output. -// AffineQuantize takes scale and zero point and quantizes the float value to -// quantized output, in int8 or uint8 format. -TfLiteRegistration* Register_QUANTIZE() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/quantize::Prepare, - /*invoke=*/quantize::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_QUANTIZE() { + return {/*init=*/Init, + /*free=*/nullptr, + /*prepare=*/Prepare, + /*invoke=*/Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -} // namespace micro -} // namespace ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/reduce.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/reduce.cc index 09894ddc..8c60269c 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/reduce.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/reduce.cc @@ -18,9 +18,12 @@ limitations under the License. #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/integer_ops/mean.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/internal/types.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/micro_utils.h" namespace tflite { namespace ops { @@ -30,10 +33,27 @@ namespace reduce { constexpr int kMaxNumberOfAxis = 4; constexpr int kMaxNumberOfReducedAxis = 2; +struct OpData { + int32_t multiplier; + int shift; + int temp_buffer_idx; + int resolved_axis_idx; + int input_zp; + float input_scale; + int output_zp; + float output_scale; + int num_output_elements; +}; + +void* InitReduce(TfLiteContext* context, const char* buffer, size_t length) { + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + TfLiteStatus PrepareSimple(TfLiteContext* context, TfLiteNode* node) { // Inputs Tensor (dtype depends on quantization): // [0] = Input // [1] = Axis + const TfLiteTensor* input = GetInput(context, node, 0); // Outputs Tensor (dtype depends on quantization): // [0] = Output @@ -44,13 +64,63 @@ TfLiteStatus PrepareSimple(TfLiteContext* context, TfLiteNode* node) { // Validate axis type const TfLiteTensor* axis = GetInput(context, node, 1); + TF_LITE_ENSURE(context, axis != nullptr); TF_LITE_ENSURE_TYPES_EQ(context, axis->type, kTfLiteInt32); + + if (input->type == kTfLiteInt8) { + OpData* data = static_cast(node->user_data); + const TfLiteTensor* output = GetOutput(context, node, 0); + const double real_multiplier = static_cast(input->params.scale) / + static_cast(output->params.scale); + QuantizeMultiplier(real_multiplier, &data->multiplier, &data->shift); + } + + return kTfLiteOk; +} + +TfLiteStatus PrepareMax(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_OK(context, PrepareSimple(context, node)); + + OpData* op_data = static_cast(node->user_data); + const TfLiteTensor* input = GetInput(context, node, 0); + const TfLiteTensor* output = GetOutput(context, node, 0); + const TfLiteTensor* axis = GetInput(context, node, 1); + + op_data->input_scale = input->params.scale; + op_data->output_scale = output->params.scale; + op_data->num_output_elements = NumElements(output); + + context->RequestScratchBufferInArena(context, sizeof(int) * input->dims->size, + &op_data->temp_buffer_idx); + context->RequestScratchBufferInArena( + context, sizeof(int) * static_cast(ElementCount(*axis->dims)), + &op_data->resolved_axis_idx); + return kTfLiteOk; } TfLiteStatus PrepareMeanOrSum(TfLiteContext* context, TfLiteNode* node) { + const TfLiteTensor* input = GetInput(context, node, 0); + OpData* op_data = reinterpret_cast(node->user_data); + const TfLiteTensor* output = GetOutput(context, node, 0); + if (input->type == kTfLiteInt8) { + const double real_multiplier = static_cast(input->params.scale) / + static_cast(output->params.scale); + QuantizeMultiplier(real_multiplier, &op_data->multiplier, &op_data->shift); + } + + int output_size = NumElements(output); + if (input->type == kTfLiteInt8 || input->type == kTfLiteUInt8) { + context->RequestScratchBufferInArena(context, output_size * sizeof(int32_t), + &op_data->temp_buffer_idx); + op_data->input_zp = input->params.zero_point; + op_data->input_scale = input->params.scale; + op_data->output_zp = output->params.zero_point; + op_data->output_scale = output->params.scale; + } + TF_LITE_ENSURE_OK(context, PrepareSimple(context, node)); - // TODO(b/144955155): Support uint8(b/144955155) and int8(b/144955018) + // TODO(b/144955155): Support uint8_t(b/144955155) and int8_t(b/144955018) return kTfLiteOk; } @@ -58,7 +128,7 @@ void ResolveAxis(const int* axis_data, int axis_count, tflite::MeanParams* op_params) { int i = 0; for (; i < axis_count; ++i) { - op_params->axis[i] = static_cast(axis_data[i]); + op_params->axis[i] = static_cast(axis_data[i]); } for (; i < 4; ++i) { op_params->axis[i] = 1; @@ -67,69 +137,206 @@ void ResolveAxis(const int* axis_data, int axis_count, } TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, 0); - const TfLiteTensor* axis = GetInput(context, node, 1); - TfLiteTensor* output = GetOutput(context, node, 0); + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 1); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); TfLiteReducerParams* params = reinterpret_cast(node->builtin_data); + OpData* op_data = reinterpret_cast(node->user_data); - int num_axis = static_cast(NumElements(axis)); + int num_axis = static_cast(ElementCount(*axis->dims)); int temp_index[kMaxNumberOfAxis]; int resolved_axis[kMaxNumberOfReducedAxis]; + tflite::MeanParams op_params; + ResolveAxis(tflite::micro::GetTensorData(axis), num_axis, &op_params); + + // Special case mean implementation exists for 4D mean across axes 1 and 2. + bool special_case_4d_axes_1_and_2 = + input->dims->size == 4 && op_params.axis_count == 2 && + ((op_params.axis[0] == 1 && op_params.axis[1] == 2) || + (op_params.axis[0] == 2 && op_params.axis[1] == 1)); + switch (input->type) { case kTfLiteFloat32: { - tflite::MeanParams op_params; - ResolveAxis(GetTensorData(axis), num_axis, &op_params); - // TODO(b/146571391): Support only 4D Input and 2D Axis for Mean until - // scratch tensor allocation has been implemented in (b/132070898) - bool is_valid_inputs = - (NumDimensions(input) == 4 && op_params.axis_count == 2 && - ((op_params.axis[0] == 1 && op_params.axis[1] == 2) || - (op_params.axis[0] == 2 && op_params.axis[1] == 1))); - TF_LITE_ENSURE_MSG( - context, is_valid_inputs == true, - "Number of Input " - "dimensions != 4 OR the Axis is not either [1, 2] or [2, 1]"); - // TODO(b/139102329): Handle the below special case in the combined - // reference method. // Defer to specialized implementation for 4D Mean across axes 1 & 2. - if (params->keep_dims) { - reference_ops::Mean(op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(output), - GetTensorData(output)); + if (params->keep_dims && special_case_4d_axes_1_and_2) { + reference_ops::Mean(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { TF_LITE_ENSURE( context, - reference_ops::Mean(GetTensorData(input), input->dims->data, - input->dims->size, GetTensorData(output), + reference_ops::Mean( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_index, resolved_axis, + tflite::micro::GetTensorData(output))); + } + } break; + case kTfLiteInt8: { + // Defer to specialized implementation for 4D Mean across axes 1 & 2. + if (params->keep_dims && special_case_4d_axes_1_and_2) { + reference_integer_ops::Mean( + op_params, op_data->multiplier, op_data->shift, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), op_data->input_zp, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), op_data->output_zp); + } else if (op_data->input_zp == op_data->output_zp && + op_data->input_scale == op_data->output_scale) { + int32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::Mean( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_index, resolved_axis, temp_buffer)); + } else { + int32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::QuantizedMeanOrSum( + tflite::micro::GetTensorData(input), op_data->input_zp, + op_data->input_scale, input->dims->data, input->dims->size, + tflite::micro::GetTensorData(output), + op_data->output_zp, op_data->output_scale, output->dims->data, + output->dims->size, tflite::micro::GetTensorData(axis), + num_axis, params->keep_dims, temp_index, resolved_axis, + temp_buffer, false)); + } + } break; + case kTfLiteUInt8: { + // Defer to specialized implementation for 4D Mean across axes 1 & 2. + if (params->keep_dims && special_case_4d_axes_1_and_2) { + reference_ops::Mean(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + op_data->input_zp, op_data->input_scale, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), + op_data->output_zp, op_data->output_scale); + } else if (op_data->input_zp == op_data->output_zp && + op_data->input_scale == op_data->output_scale) { + uint32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::Mean(tflite::micro::GetTensorData(input), + input->dims->data, input->dims->size, + tflite::micro::GetTensorData(output), output->dims->data, output->dims->size, - GetTensorData(axis), num_axis, - params->keep_dims, temp_index, resolved_axis, - GetTensorData(output))); + tflite::micro::GetTensorData(axis), + num_axis, params->keep_dims, temp_index, + resolved_axis, temp_buffer)); + } else { + uint32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::QuantizedMeanOrSum( + tflite::micro::GetTensorData(input), op_data->input_zp, + op_data->input_scale, input->dims->data, input->dims->size, + tflite::micro::GetTensorData(output), + op_data->output_zp, op_data->output_scale, output->dims->data, + output->dims->size, tflite::micro::GetTensorData(axis), + num_axis, params->keep_dims, temp_index, resolved_axis, + temp_buffer, false)); } } break; default: - // TODO(b/144955155): Support uint8(b/144955155) and int8(b/144955018) TF_LITE_ENSURE_MSG(context, false, - "Currently, only float32 input type " + "Currently, only float32, int8 or uint8 input type " "is supported."); } return kTfLiteOk; } + +TfLiteStatus EvalMax(TfLiteContext* context, TfLiteNode* node) { + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 1); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); + TfLiteReducerParams* params = + static_cast(node->builtin_data); + OpData* op_data = static_cast(node->user_data); + + // Interpret an axis tensor with null dimensions as a scalar + int num_axis = static_cast(ElementCount(*axis->dims)); + int* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + int* resolved_axis = static_cast( + context->GetScratchBuffer(context, op_data->resolved_axis_idx)); + switch (input->type) { + case kTfLiteFloat32: + TF_LITE_ENSURE( + context, + reference_ops::ReduceGeneric( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_buffer, resolved_axis, + std::numeric_limits::lowest(), + [](const float current, const float in) -> float { + return (in > current) ? in : current; + })); + break; + case kTfLiteInt8: + TF_LITE_ENSURE_EQ(context, static_cast(op_data->input_scale), + static_cast(op_data->output_scale)); + TF_LITE_ENSURE_EQ(context, op_data->input_zp, op_data->output_zp); + TF_LITE_ENSURE( + context, + reference_ops::ReduceGeneric( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_buffer, resolved_axis, + std::numeric_limits::lowest(), + [](const int8_t current, const int8_t in) -> int8_t { + return (in > current) ? in : current; + })); + break; + default: + TF_LITE_KERNEL_LOG(context, + "Only float32 and int8 types are supported.\n"); + return kTfLiteError; + } + return kTfLiteOk; +} + } // namespace reduce -TfLiteRegistration* Register_MEAN() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/reduce::PrepareMeanOrSum, - /*invoke=*/reduce::EvalMean, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_MEAN() { + return {/*init=*/reduce::InitReduce, + /*free=*/nullptr, + /*prepare=*/reduce::PrepareMeanOrSum, + /*invoke=*/reduce::EvalMean, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } + +TfLiteRegistration Register_REDUCE_MAX() { + return {/*init=*/reduce::InitReduce, + /*free=*/nullptr, + /*prepare=*/reduce::PrepareMax, + /*invoke=*/reduce::EvalMax, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; +} + } // namespace micro } // namespace ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/reshape.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/reshape.cc index 407682a6..8e47e2a0 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/reshape.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/reshape.cc @@ -18,6 +18,9 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/memory_helpers.h" +#include "tensorflow/lite/micro/micro_utils.h" namespace tflite { namespace ops { @@ -29,7 +32,9 @@ constexpr int kOutputTensor = 0; TfLiteStatus ReshapeOutput(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); // Tensorflow's Reshape allows one of the shape components to have the // special -1 value, meaning it will be calculated automatically based on the // input. Here we calculate what that dimension should be so that the number @@ -61,7 +66,7 @@ TfLiteStatus ReshapeOutput(TfLiteContext* context, TfLiteNode* node) { num_output_elements *= output_shape->data[stretch_dim]; } - TF_LITE_ENSURE_EQ(context, input->type, output->type); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); TF_LITE_ENSURE_EQ(context, num_input_elements, num_output_elements); return kTfLiteOk; } @@ -74,13 +79,21 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + // TODO(b/162522304): storing input bytes in OpData increases some models + // significantly, possibly due to alignment issues. + size_t input_bytes; + TF_LITE_ENSURE_STATUS(TfLiteTypeSizeOf(input->type, &input_bytes)); + input_bytes *= ElementCount(*input->dims); // Do nothing for in-place reshape. if (input->data.raw != output->data.raw) { // Otherwise perform reshape with copy. - for (size_t i = 0; i < input->bytes; ++i) { + for (size_t i = 0; i < input_bytes; ++i) { output->data.raw[i] = input->data.raw[i]; } } @@ -89,16 +102,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace reshape -TfLiteRegistration* Register_RESHAPE() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/reshape::Prepare, - /*invoke=*/reshape::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_RESHAPE() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/reshape::Prepare, + /*invoke=*/reshape::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc index 9487e33c..971de83c 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -31,7 +32,6 @@ constexpr int kSizeTensor = 1; constexpr int kOutputTensor = 0; TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { -#if defined(DEBUG) TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); @@ -49,11 +49,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { output->type = input->type; if (!IsConstantTensor(size)) { - TF_LITE_KERNEL_LOG(context, - "Dynamic tensors are unsupported in tfmicro."); + TF_LITE_KERNEL_LOG(context, "Dynamic tensors are unsupported in tfmicro."); return kTfLiteError; } -#endif return kTfLiteOk; } @@ -61,9 +59,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { auto* params = reinterpret_cast(node->builtin_data); - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - const TfLiteTensor* size = GetInput(context, node, kSizeTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + const TfLiteEvalTensor* size = + tflite::micro::GetEvalInput(context, node, kSizeTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); tflite::ResizeNearestNeighborParams op_params; op_params.align_corners = params->align_corners; @@ -71,22 +72,31 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { if (output->type == kTfLiteFloat32) { reference_ops::ResizeNearestNeighbor( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(size), GetTensorData(size), - GetTensorShape(output), GetTensorData(output)); + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(size), + tflite::micro::GetTensorData(size), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else if (output->type == kTfLiteUInt8) { reference_ops::ResizeNearestNeighbor( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(size), GetTensorData(size), - GetTensorShape(output), GetTensorData(output)); + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(size), + tflite::micro::GetTensorData(size), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else if (output->type == kTfLiteInt8) { reference_ops::ResizeNearestNeighbor( - op_params, GetTensorShape(input), GetTensorData(input), - GetTensorShape(size), GetTensorData(size), - GetTensorShape(output), GetTensorData(output)); + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(size), + tflite::micro::GetTensorData(size), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { TF_LITE_KERNEL_LOG(context, - "Output type is %d, requires float, uint8 or int8.", + "Output type is %d, requires float, uint8_t or int8_t.", output->type); return kTfLiteError; } @@ -95,16 +105,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } } // namespace resize_nearest_neighbor -TfLiteRegistration* Register_RESIZE_NEAREST_NEIGHBOR() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/resize_nearest_neighbor::Prepare, - /*invoke=*/resize_nearest_neighbor::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_RESIZE_NEAREST_NEIGHBOR() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/resize_nearest_neighbor::Prepare, + /*invoke=*/resize_nearest_neighbor::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/round.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/round.cc index b88c9fe0..5804016b 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/round.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/round.cc @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -29,11 +30,13 @@ constexpr int kOutputTensor = 0; TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); - TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32); - TF_LITE_ENSURE_EQ(context, output->type, input->type); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, kTfLiteFloat32); + TF_LITE_ENSURE_TYPES_EQ(context, output->type, input->type); TF_LITE_ENSURE_EQ(context, output->bytes, input->bytes); TF_LITE_ENSURE_EQ(context, output->dims->size, input->dims->size); for (int i = 0; i < output->dims->size; ++i) { @@ -43,26 +46,29 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); - reference_ops::Round(GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); + reference_ops::Round(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); return kTfLiteOk; } } // namespace round -TfLiteRegistration* Register_ROUND() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/round::Prepare, - /*invoke=*/round::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_ROUND() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/round::Prepare, + /*invoke=*/round::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/shape.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/shape.cc new file mode 100644 index 00000000..df962f62 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/shape.cc @@ -0,0 +1,73 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/memory_helpers.h" +#include "tensorflow/lite/micro/micro_utils.h" + +namespace tflite { + +namespace { +constexpr int kInputTensor = 0; +constexpr int kOutputTensor = 0; + +void ExtractShape(const TfLiteEvalTensor* input, int32_t* output_data) { + for (int i = 0; i < input->dims->size; ++i) { + output_data[i] = input->dims->data[i]; + } +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + return kTfLiteOk; +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + if (output->type != kTfLiteInt32) { + TF_LITE_KERNEL_LOG(context, "Output type %s (%d) not supported.", + TfLiteTypeGetName(output->type), output->type); + return kTfLiteError; + } else { + ExtractShape(input, tflite::micro::GetTensorData(output)); + } + + return kTfLiteOk; +} + +} // namespace + +TfLiteRegistration Register_SHAPE() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/Prepare, + /*invoke=*/Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; +} + +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/softmax.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/softmax.cc index 3d8e7cd8..c96fa561 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/softmax.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/softmax.cc @@ -22,29 +22,35 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { -namespace ops { -namespace micro { -namespace activations { namespace { +// Softmax parameter data that persists in user_data +static constexpr int kInt16LUTArraySize = 513; + TfLiteStatus CalculateSoftmaxParams(TfLiteContext* context, const TfLiteTensor* input, TfLiteTensor* output, const TfLiteSoftmaxParams* params, SoftmaxParams* op_data) { - if (input->type == kTfLiteUInt8 || input->type == kTfLiteInt8) { + if (input->type == kTfLiteUInt8 || input->type == kTfLiteInt8 || + input->type == kTfLiteInt16) { if (input->type == kTfLiteUInt8) { TF_LITE_ENSURE_TYPES_EQ(context, output->type, kTfLiteUInt8); TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); - } else { + } else if (input->type == kTfLiteInt16) { + TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); + TF_LITE_ENSURE_NEAR(context, output->params.scale, 1.f / 32768, + (0.001f * 1.f / 32768)); + } else { // input->type == kTfLiteInt8 TF_LITE_ENSURE_TYPES_EQ(context, input->type, kTfLiteInt8); if (output->type == kTfLiteInt16) { TF_LITE_ENSURE_EQ(context, output->params.zero_point, -32768); - // NOTE: Current int16 softmax output does not require symmetric scaling - // - so no need to verify scale here. - } else { + TF_LITE_ENSURE_NEAR(context, output->params.scale, 1.f / 65536, + (0.001f * 1.f / 65536)); + } else { // output->type == kTfLiteint8 TF_LITE_ENSURE_TYPES_EQ(context, output->type, kTfLiteInt8); TF_LITE_ENSURE_EQ(context, output->params.zero_point, -128); TF_LITE_ENSURE(context, output->params.scale == 1.f / 256); @@ -53,15 +59,28 @@ TfLiteStatus CalculateSoftmaxParams(TfLiteContext* context, static const int kScaledDiffIntegerBits = 5; - int input_left_shift; - tflite::PreprocessSoftmaxScaling( - static_cast(params->beta), - static_cast(input->params.scale), kScaledDiffIntegerBits, - &op_data->input_multiplier, &input_left_shift); - op_data->input_left_shift = input_left_shift; - op_data->diff_min = - -1.0 * tflite::CalculateInputRadius(kScaledDiffIntegerBits, - op_data->input_left_shift); + // Calculate input_multiplier and input_left_shift + if (input->type == kTfLiteInt16) { + int input_left_shift; + double input_scale_beta_rescale = + static_cast(input->params.scale) * + static_cast(params->beta) / + (10.0 / 65535.0); // scale the input_diff such that [-65535, 0] + // correspond to [-10.0, 0.0] + QuantizeMultiplier(input_scale_beta_rescale, &op_data->input_multiplier, + &input_left_shift); + op_data->input_left_shift = input_left_shift; + } else { + int input_left_shift; + tflite::PreprocessSoftmaxScaling( + static_cast(params->beta), + static_cast(input->params.scale), kScaledDiffIntegerBits, + &op_data->input_multiplier, &input_left_shift); + op_data->input_left_shift = input_left_shift; + op_data->diff_min = + -1.0 * tflite::CalculateInputRadius(kScaledDiffIntegerBits, + op_data->input_left_shift); + } } else { TF_LITE_ENSURE_TYPES_EQ(context, input->type, kTfLiteFloat32); TF_LITE_ENSURE_TYPES_EQ(context, output->type, kTfLiteFloat32); @@ -70,53 +89,106 @@ TfLiteStatus CalculateSoftmaxParams(TfLiteContext* context, return kTfLiteOk; } -} // namespace +// Takes a tensor and performs softmax along the last dimension. +void SoftmaxFloat(const TfLiteEvalTensor* input, TfLiteEvalTensor* output, + const SoftmaxParams& op_data) { + tflite::reference_ops::Softmax(op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +} + +void SoftmaxQuantized(const TfLiteEvalTensor* input, TfLiteEvalTensor* output, + const SoftmaxParams& op_data) { + if (input->type == kTfLiteUInt8) { + tflite::reference_ops::Softmax( + op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } else if (input->type == kTfLiteInt8) { + if (output->type == kTfLiteInt16) { + tflite::reference_ops::Softmax( + op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } else { + tflite::reference_ops::Softmax( + op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } + } else { + tflite::reference_ops::SoftmaxInt16( + op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } +} + +void* SoftmaxInit(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(SoftmaxParams)); +} TfLiteStatus SoftmaxPrepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); const TfLiteTensor* input = GetInput(context, node, 0); + TF_LITE_ENSURE(context, input != nullptr); TF_LITE_ENSURE(context, NumDimensions(input) >= 1); + TfLiteTensor* output = GetOutput(context, node, 0); + TF_LITE_ENSURE(context, output != nullptr); - return kTfLiteOk; -} - -// Takes a tensor and performs softmax along the last dimension. -void SoftmaxFloat(const TfLiteTensor* input, TfLiteTensor* output, - const SoftmaxParams& op_data) { - tflite::reference_ops::Softmax( - op_data, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); -} - -void SoftmaxQuantized(const TfLiteTensor* input, TfLiteTensor* output, - const SoftmaxParams& op_data) { - if (input->type == kTfLiteUInt8) { - tflite::reference_ops::Softmax( - op_data, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); - } else { - if (output->type == kTfLiteInt16) { - tflite::reference_ops::Softmax( - op_data, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); - } else { - tflite::reference_ops::Softmax( - op_data, GetTensorShape(input), GetTensorData(input), - GetTensorShape(output), GetTensorData(output)); - } + TF_LITE_ENSURE(context, node->user_data != nullptr); + SoftmaxParams* op_data = static_cast(node->user_data); + // Only allocate LUTs for KTfLiteInt16 data type + if (input->type == kTfLiteInt16) { + void* raw_exp_lut = context->AllocatePersistentBuffer( + context, sizeof(int16_t) * kInt16LUTArraySize); + TF_LITE_ENSURE(context, raw_exp_lut != nullptr); + op_data->exp_lut = reinterpret_cast(raw_exp_lut); + void* one_over_one_plus_x_lut = context->AllocatePersistentBuffer( + context, sizeof(int16_t) * kInt16LUTArraySize); + TF_LITE_ENSURE(context, one_over_one_plus_x_lut != nullptr); + op_data->one_over_one_plus_x_lut = + reinterpret_cast(one_over_one_plus_x_lut); } + + if (output->type == kTfLiteInt16) { + TF_LITE_ENSURE(context, input->type == kTfLiteInt8 || + input->type == kTfLiteUInt8 || + input->type == kTfLiteInt16); + } else { + TF_LITE_ENSURE_EQ(context, input->type, output->type); + } + + // Populate LUT if required + if (input->type == kTfLiteInt16) { + TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); + // exp LUT only used on negative values + // we consider exp(-10.0) is insignificant to accumulation + gen_lut([](float value) { return std::exp(value); }, -10.0f, 0.0f, + op_data->exp_lut, kInt16LUTArraySize); + gen_lut([](float value) { return 1.0f / (1.0f + value); }, 0.0f, 1.0f, + op_data->one_over_one_plus_x_lut, kInt16LUTArraySize); + op_data->zero_point = output->params.zero_point; + op_data->scale = output->params.scale; + } + + auto* params = static_cast(node->builtin_data); + return CalculateSoftmaxParams(context, input, output, params, op_data); } TfLiteStatus SoftmaxEval(TfLiteContext* context, TfLiteNode* node) { - auto* params = static_cast(node->builtin_data); + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); - const TfLiteTensor* input = GetInput(context, node, 0); - TfLiteTensor* output = GetOutput(context, node, 0); - - SoftmaxParams op_data; - TF_LITE_ENSURE_STATUS( - CalculateSoftmaxParams(context, input, output, params, &op_data)); + TFLITE_DCHECK(node->user_data != nullptr); + SoftmaxParams op_data = *static_cast(node->user_data); switch (input->type) { case kTfLiteFloat32: { @@ -124,7 +196,8 @@ TfLiteStatus SoftmaxEval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } case kTfLiteInt8: - case kTfLiteUInt8: { + case kTfLiteUInt8: + case kTfLiteInt16: { SoftmaxQuantized(input, output, op_data); return kTfLiteOk; } @@ -134,20 +207,17 @@ TfLiteStatus SoftmaxEval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteError; } } -} // namespace activations +} // namespace -TfLiteRegistration* Register_SOFTMAX() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/activations::SoftmaxPrepare, - /*invoke=*/activations::SoftmaxEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_SOFTMAX() { + return {/*init=*/SoftmaxInit, + /*free=*/nullptr, + /*prepare=*/SoftmaxPrepare, + /*invoke=*/SoftmaxEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -} // namespace micro -} // namespace ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/split.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/split.cc index 94b1508d..a1236d71 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/split.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/split.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -25,10 +26,11 @@ namespace split { template TfLiteStatus SplitImpl(TfLiteContext* context, TfLiteNode* node, - const TfLiteTensor* input, int axis_value) { + const TfLiteEvalTensor* input, int axis_value) { const int output_count = NumOutputs(node); const TfLiteIntArray* input_dims = input->dims; - const TfLiteTensor* output0 = GetOutput(context, node, 0); + const TfLiteEvalTensor* output0 = + tflite::micro::GetEvalOutput(context, node, 0); const TfLiteIntArray* output_dims = output0->dims; const int split_dimensions = input_dims->size; @@ -50,11 +52,11 @@ TfLiteStatus SplitImpl(TfLiteContext* context, TfLiteNode* node, base_inner_size *= input_dims->data[i]; } - const T* input_ptr = GetTensorData(input); + const T* input_ptr = tflite::micro::GetTensorData(input); for (int k = 0; k < outer_size; ++k) { for (int i = 0; i < output_count; ++i) { - TfLiteTensor* t = GetOutput(context, node, i); - T* output_data = GetTensorData(t); + TfLiteEvalTensor* t = tflite::micro::GetEvalOutput(context, node, i); + T* output_data = tflite::micro::GetTensorData(t); const int copy_size = output_dims->data[axis] * base_inner_size; T* output_ptr = output_data + k * copy_size; for (int j = 0; j < copy_size; ++j) output_ptr[j] = input_ptr[j]; @@ -65,23 +67,29 @@ TfLiteStatus SplitImpl(TfLiteContext* context, TfLiteNode* node, return kTfLiteOk; } -TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* axis = GetInput(context, node, 0); - const TfLiteTensor* input = GetInput(context, node, 1); + TF_LITE_ENSURE(context, axis != nullptr); // Dynamic output tensors are needed if axis tensor is not constant. // But Micro doesn't support dynamic memory allocation, so we only support // constant axis tensor for now. TF_LITE_ENSURE_MSG(context, IsConstantTensor(axis), "Non constant axis tensor not supported"); + return kTfLiteOk; +} - int axis_value = GetTensorData(axis)[0]; +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 0); + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 1); + + int axis_value = tflite::micro::GetTensorData(axis)[0]; if (axis_value < 0) { - axis_value += NumDimensions(input); + axis_value += input->dims->size; } TF_LITE_ENSURE(context, axis_value >= 0); - TF_LITE_ENSURE(context, axis_value < NumDimensions(input)); + TF_LITE_ENSURE(context, axis_value < input->dims->size); switch (input->type) { case kTfLiteFloat32: { @@ -111,16 +119,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace split -TfLiteRegistration* Register_SPLIT() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/split::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_SPLIT() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/split::Prepare, + /*invoke=*/split::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/split_v.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/split_v.cc new file mode 100644 index 00000000..c2a01149 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/split_v.cc @@ -0,0 +1,135 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" + +namespace tflite { +namespace ops { +namespace micro { +namespace split_v { + +template +TfLiteStatus SplitImpl(TfLiteContext* context, TfLiteNode* node, + const TfLiteEvalTensor* input, int axis_value) { + const TfLiteIntArray* input_dims = input->dims; + const TfLiteEvalTensor* output0 = + tflite::micro::GetEvalOutput(context, node, 0); + + const int split_dimensions = input_dims->size; + + TFLITE_DCHECK_LT(axis_value, split_dimensions); + TFLITE_DCHECK_EQ(output0->dims->size, split_dimensions); + + int64_t split_size = 0; + const int output_count = NumOutputs(node); + for (int i = 0; i < output_count; i++) { + split_size += + tflite::micro::GetEvalOutput(context, node, i)->dims->data[axis_value]; + } + TFLITE_DCHECK_EQ(split_size, input_dims->data[axis_value]); + int64_t outer_size = 1; + for (int i = 0; i < axis_value; ++i) { + outer_size *= input_dims->data[i]; + } + + int64_t base_inner_size = 1; + for (int i = axis_value + 1; i < split_dimensions; ++i) { + base_inner_size *= input_dims->data[i]; + } + + const T* input_ptr = tflite::micro::GetTensorData(input); + for (int k = 0; k < outer_size; ++k) { + for (int i = 0; i < output_count; ++i) { + TfLiteEvalTensor* output_tensor = + tflite::micro::GetEvalOutput(context, node, i); + T* output_data = tflite::micro::GetTensorData(output_tensor); + const int copy_size = + output_tensor->dims->data[axis_value] * base_inner_size; + T* output_ptr = output_data + k * copy_size; + for (int j = 0; j < copy_size; ++j) output_ptr[j] = input_ptr[j]; + input_ptr += copy_size; + } + } + + return kTfLiteOk; +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 3); + + // Dynamic output tensors are needed if axis tensor is not constant. + // But Micro doesn't support dynamic memory allocation, so we only support + // constant axis tensor for now. + const TfLiteTensor* axis = GetInput(context, node, 2); + TF_LITE_ENSURE_MSG(context, IsConstantTensor(axis), + "Non constant axis tensor not supported"); + + return kTfLiteOk; +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 2); + + int axis_value = tflite::micro::GetTensorData(axis)[0]; + if (axis_value < 0) { + axis_value += input->dims->size; + } + + TF_LITE_ENSURE(context, axis_value >= 0); + TF_LITE_ENSURE(context, axis_value < input->dims->size); + + switch (input->type) { + case kTfLiteFloat32: { + return SplitImpl(context, node, input, axis_value); + } + case kTfLiteInt8: { + return SplitImpl(context, node, input, axis_value); + } + case kTfLiteInt16: { + return SplitImpl(context, node, input, axis_value); + } + case kTfLiteInt32: { + return SplitImpl(context, node, input, axis_value); + } + default: + TF_LITE_KERNEL_LOG(context, "Type %s currently not supported.", + TfLiteTypeGetName(input->type)); + return kTfLiteError; + } + return kTfLiteOk; +} + +} // namespace split_v + +TfLiteRegistration Register_SPLIT_V() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/split_v::Prepare, + /*invoke=*/split_v::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; +} + +} // namespace micro +} // namespace ops +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/strided_slice.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/strided_slice.cc index df6c4294..2dbe6e1d 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/strided_slice.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/strided_slice.cc @@ -15,23 +15,20 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/reference/strided_slice.h" #include +#include #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { namespace micro { namespace strided_slice { -enum KernelType { - kReference, - // TODO(soroosh): add kGenericOptimized -}; - constexpr int kInputTensor = 0; constexpr int kBeginTensor = 1; constexpr int kEndTensor = 2; @@ -120,64 +117,74 @@ TfLiteStatus CheckOutputSize(TfLiteContext* context, return kTfLiteOk; } +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(StridedSliceParams)); +} + TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + StridedSliceParams* op_params = + static_cast(node->user_data); TF_LITE_ENSURE_EQ(context, NumInputs(node), 4); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); StridedSliceContext op_context(context, node); TF_LITE_ENSURE_MSG(context, op_context.dims <= kMaxDim, "input dim should not exceed 4"); + auto params = BuildStridedSliceParams(&op_context); + memcpy(op_params, ¶ms, sizeof(StridedSliceParams)); return CheckOutputSize(context, &op_context); } -template TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - StridedSliceContext op_context(context, node); - auto op_params = BuildStridedSliceParams(&op_context); + TFLITE_DCHECK(node->user_data != nullptr); + const StridedSliceParams& op_params = + *(static_cast(node->user_data)); -#define TF_LITE_STRIDED_SLICE(kernel_type, data_type) \ - kernel_type::StridedSlice(op_params, GetTensorShape(op_context.input), \ - GetTensorData(op_context.input), \ - GetTensorShape(op_context.output), \ - GetTensorData(op_context.output)) - - switch (op_context.input->type) { + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + switch (output->type) { case kTfLiteFloat32: - if (kernel_type == kReference) { - TF_LITE_STRIDED_SLICE(reference_ops, float); - } + reference_ops::StridedSlice(op_params, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; case kTfLiteUInt8: - if (kernel_type == kReference) { - TF_LITE_STRIDED_SLICE(reference_ops, uint8_t); - } + reference_ops::StridedSlice( + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; case kTfLiteInt8: - if (kernel_type == kReference) { - TF_LITE_STRIDED_SLICE(reference_ops, int8_t); - } + reference_ops::StridedSlice(op_params, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); break; default: TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", - TfLiteTypeGetName(op_context.input->type), - op_context.input->type); + TfLiteTypeGetName(input->type), input->type); return kTfLiteError; } -#undef TF_LITE_STRIDED_SLICE return kTfLiteOk; } } // namespace strided_slice -TfLiteRegistration* Register_STRIDED_SLICE() { - static TfLiteRegistration r = { - /*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/strided_slice::Prepare, - /*invoke=*/strided_slice::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_STRIDED_SLICE() { + return {/*init=*/strided_slice::Init, + /*free=*/nullptr, + /*prepare=*/strided_slice::Prepare, + /*invoke=*/strided_slice::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/sub.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/sub.cc index f27dcdc2..2cc61a9b 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/sub.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/sub.cc @@ -21,8 +21,10 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/quantization_util.h" #include "tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/internal/types.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -40,18 +42,18 @@ struct OpData { // and the special 16-bit -> 16bit quantized path int input1_shift; int input2_shift; - int32 output_activation_min; - int32 output_activation_max; + int32_t output_activation_min; + int32_t output_activation_max; // These fields are used only in the general 8-bit -> 8bit quantized path - int32 input1_multiplier; - int32 input2_multiplier; - int32 output_multiplier; + int32_t input1_multiplier; + int32_t input2_multiplier; + int32_t output_multiplier; int output_shift; int left_shift; - int32 input1_offset; - int32 input2_offset; - int32 output_offset; + int32_t input1_offset; + int32_t input2_offset; + int32_t output_offset; }; TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteSubParams* params, @@ -93,31 +95,62 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteSubParams* params, return kTfLiteOk; } +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); + + OpData* data = static_cast(node->user_data); + auto* params = reinterpret_cast(node->builtin_data); + + const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); + TF_LITE_ENSURE(context, input1 != nullptr); + const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); + TF_LITE_ENSURE(context, input2 != nullptr); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + TF_LITE_ENSURE_STATUS( + CalculateOpData(context, params, input1, input2, output, data)); + return kTfLiteOk; +} + void EvalSub(TfLiteContext* context, TfLiteNode* node, TfLiteSubParams* params, - const OpData* data, const TfLiteTensor* input1, - const TfLiteTensor* input2, TfLiteTensor* output) { + const OpData* data, const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, TfLiteEvalTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); tflite::ArithmeticParams op_params; SetActivationParams(output_activation_min, output_activation_max, &op_params); -#define TF_LITE_SUB(opname) \ - opname(op_params, GetTensorShape(input1), GetTensorData(input1), \ - GetTensorShape(input2), GetTensorData(input2), \ - GetTensorShape(output), GetTensorData(output)) if (data->requires_broadcast) { - TF_LITE_SUB(tflite::reference_ops::BroadcastSubSlow); + tflite::reference_ops::BroadcastSubSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_SUB(tflite::reference_ops::SubWithActivation); + tflite::reference_ops::SubWithActivation( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } -#undef TF_LITE_SUB } TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, TfLiteSubParams* params, const OpData* data, - const TfLiteTensor* input1, - const TfLiteTensor* input2, - TfLiteTensor* output) { + const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, + TfLiteEvalTensor* output) { if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8) { tflite::ArithmeticParams op_params; op_params.left_shift = data->left_shift; @@ -133,25 +166,46 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, SetActivationParams(data->output_activation_min, data->output_activation_max, &op_params); bool need_broadcast = reference_ops::ProcessBroadcastShapes( - GetTensorShape(input1), GetTensorShape(input2), &op_params); -#define TF_LITE_SUB(opname, dtype) \ - opname(op_params, GetTensorShape(input1), GetTensorData(input1), \ - GetTensorShape(input2), GetTensorData(input2), \ - GetTensorShape(output), GetTensorData(output)); + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorShape(input2), &op_params); + if (output->type == kTfLiteInt8) { if (need_broadcast) { - TF_LITE_SUB(tflite::reference_ops::BroadcastSubSlow, int8_t); + tflite::reference_ops::BroadcastSubSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_SUB(tflite::reference_ops::Sub, int8_t); + tflite::reference_ops::Sub( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } else { if (need_broadcast) { - TF_LITE_SUB(tflite::reference_ops::BroadcastSubSlow, uint8_t); + tflite::reference_ops::BroadcastSubSlow( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } else { - TF_LITE_SUB(tflite::reference_ops::Sub, uint8_t); + tflite::reference_ops::Sub( + op_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); } } -#undef TF_LITE_SUB } return kTfLiteOk; @@ -160,13 +214,15 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { auto* params = reinterpret_cast(node->builtin_data); - const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); - const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); - OpData data; - TF_LITE_ENSURE_STATUS( - CalculateOpData(context, params, input1, input2, output, &data)); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData& data = *(static_cast(node->user_data)); if (output->type == kTfLiteFloat32) { EvalSub(context, node, params, &data, input1, input2, output); @@ -184,16 +240,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace sub -TfLiteRegistration* Register_SUB() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/sub::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_SUB() { + return {/*init=*/sub::Init, + /*free=*/nullptr, + /*prepare=*/sub::Prepare, + /*invoke=*/sub::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/svdf.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/svdf.cc index 8c33fde5..764fdc1b 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/svdf.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/svdf.cc @@ -23,25 +23,38 @@ limitations under the License. #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" #include "tensorflow/lite/micro/kernels/activation_utils.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" #include "tensorflow/lite/micro/micro_utils.h" namespace tflite { -namespace ops { -namespace micro { -namespace svdf { namespace { struct OpData { - int32 effective_scale_1_a; - int32 effective_scale_2_a; + int32_t effective_scale_1_a; + int32_t effective_scale_2_a; // b versions of each scale are kept at int since the numbers are just the // shift value - typically between [-32, 32]. int effective_scale_1_b; int effective_scale_2_b; int scratch_tensor_index; int scratch_output_tensor_index; + + // Cached tensor zero point values for quantized operations. + int input_zero_point; + int output_zero_point; }; +// Input tensors. +constexpr int kInputTensor = 0; +constexpr int kWeightsFeatureTensor = 1; +constexpr int kWeightsTimeTensor = 2; +constexpr int kBiasTensor = 3; +// This is a variable tensor, and will be modified by this op. +constexpr int kInputActivationStateTensor = 4; + +// Output tensor. +constexpr int kOutputTensor = 0; + /** * This version of SVDF is specific to TFLite Micro. It contains the following * differences between the TFLite version: @@ -107,18 +120,19 @@ static inline void ApplyTimeWeightsBiasAndActivation( for (int b = 0; b < batch_size; ++b) { float* output_ptr_batch = output_ptr + b * num_units; for (int i = 0; i < num_units; ++i) { - *output_ptr_batch = ActivationValFloat(activation, *output_ptr_batch); + *output_ptr_batch = + tflite::ops::micro::ActivationValFloat(activation, *output_ptr_batch); ++output_ptr_batch; } } } inline void EvalFloatSVDF( - TfLiteContext* context, TfLiteNode* node, const TfLiteTensor* input, - const TfLiteTensor* weights_feature, const TfLiteTensor* weights_time, - const TfLiteTensor* bias, const TfLiteSVDFParams* params, - int scratch_tensor_index, TfLiteTensor* activation_state, - TfLiteTensor* output) { + TfLiteContext* context, TfLiteNode* node, const TfLiteEvalTensor* input, + const TfLiteEvalTensor* weights_feature, + const TfLiteEvalTensor* weights_time, const TfLiteEvalTensor* bias, + const TfLiteSVDFParams* params, int scratch_tensor_index, + TfLiteEvalTensor* activation_state, TfLiteEvalTensor* output) { const int rank = params->rank; const int batch_size = input->dims->data[0]; const int input_size = input->dims->data[1]; @@ -126,12 +140,14 @@ inline void EvalFloatSVDF( const int num_units = num_filters / rank; const int memory_size = weights_time->dims->data[1]; - const float* weights_feature_ptr = GetTensorData(weights_feature); - const float* weights_time_ptr = GetTensorData(weights_time); - const float* bias_ptr = GetTensorData(bias); - const float* input_ptr = GetTensorData(input); + const float* weights_feature_ptr = + tflite::micro::GetTensorData(weights_feature); + const float* weights_time_ptr = + tflite::micro::GetTensorData(weights_time); + const float* bias_ptr = tflite::micro::GetTensorData(bias); + const float* input_ptr = tflite::micro::GetTensorData(input); - float* state_ptr = GetTensorData(activation_state); + float* state_ptr = tflite::micro::GetTensorData(activation_state); TFLITE_DCHECK(context != nullptr); TFLITE_DCHECK(context->GetScratchBuffer != nullptr); @@ -139,7 +155,7 @@ inline void EvalFloatSVDF( float* scratch_ptr = static_cast( context->GetScratchBuffer(context, scratch_tensor_index)); - float* output_ptr = GetTensorData(output); + float* output_ptr = tflite::micro::GetTensorData(output); // Left shift the activation_state. { @@ -185,14 +201,13 @@ inline void EvalFloatSVDF( } void EvalIntegerSVDF(TfLiteContext* context, TfLiteNode* node, - const TfLiteTensor* input_tensor, - const TfLiteTensor* weights_feature_tensor, - const TfLiteTensor* weights_time_tensor, - const TfLiteTensor* bias_tensor, + const TfLiteEvalTensor* input_tensor, + const TfLiteEvalTensor* weights_feature_tensor, + const TfLiteEvalTensor* weights_time_tensor, + const TfLiteEvalTensor* bias_tensor, const TfLiteSVDFParams* params, - TfLiteTensor* activation_state_tensor, - TfLiteTensor* output_tensor, const OpData& data, - int32_t input_zp, int32_t output_zp) { + TfLiteEvalTensor* activation_state_tensor, + TfLiteEvalTensor* output_tensor, const OpData& data) { const int n_rank = params->rank; const int n_batch = input_tensor->dims->data[0]; const int n_input = input_tensor->dims->data[1]; @@ -209,7 +224,8 @@ void EvalIntegerSVDF(TfLiteContext* context, TfLiteNode* node, context->GetScratchBuffer(context, data.scratch_output_tensor_index)); // Shift states. - int16_t* const state_ptr = GetTensorData(activation_state_tensor); + int16_t* const state_ptr = + tflite::micro::GetTensorData(activation_state_tensor); // Left shift the activation_state. { @@ -225,10 +241,11 @@ void EvalIntegerSVDF(TfLiteContext* context, TfLiteNode* node, // Feature matmul. { - int16_t* state = GetTensorData(activation_state_tensor); - const int8_t* input = GetTensorData(input_tensor); + int16_t* state = + tflite::micro::GetTensorData(activation_state_tensor); + const int8_t* input = tflite::micro::GetTensorData(input_tensor); const int8_t* weight_feature = - GetTensorData(weights_feature_tensor); + tflite::micro::GetTensorData(weights_feature_tensor); const int32_t output_max = std::numeric_limits::max(); const int32_t output_min = std::numeric_limits::min(); int16_t* result_in_batch = state + (n_memory - 1); @@ -238,7 +255,8 @@ void EvalIntegerSVDF(TfLiteContext* context, TfLiteNode* node, int32_t dot_prod = 0; const int8_t* vector_in_batch = input + b * n_input; for (int c = 0; c < n_input; c++) { - dot_prod += *matrix_ptr++ * (*vector_in_batch++ - input_zp); + dot_prod += + *matrix_ptr++ * (*vector_in_batch++ - data.input_zero_point); } dot_prod = MultiplyByQuantizedMultiplier( dot_prod, data.effective_scale_1_a, data.effective_scale_1_b); @@ -261,9 +279,10 @@ void EvalIntegerSVDF(TfLiteContext* context, TfLiteNode* node, int32_t* scratch_ptr_batch = scratch_tensor + b * n_filter; // Perform batched vector dot product: - const int16_t* vector1_ptr = GetTensorData(weights_time_tensor); + const int16_t* vector1_ptr = + tflite::micro::GetTensorData(weights_time_tensor); const int16_t* vector2_ptr = - GetTensorData(activation_state_tensor) + + tflite::micro::GetTensorData(activation_state_tensor) + b * n_memory * n_filter; for (int i = 0; i < n_filter; i++) { @@ -281,7 +300,8 @@ void EvalIntegerSVDF(TfLiteContext* context, TfLiteNode* node, // Add bias. if (bias_tensor) { // Vector batch assign: - const int32_t* bias_data = GetTensorData(bias_tensor); + const int32_t* bias_data = + tflite::micro::GetTensorData(bias_tensor); for (int i = 0; i < n_batch; ++i) { int32_t* output_ptr = scratch_output_tensor + i * n_unit; const int32_t* bias_ptr = bias_data; @@ -316,34 +336,17 @@ void EvalIntegerSVDF(TfLiteContext* context, TfLiteNode* node, int32_t x1 = scratch_output_tensor[i]; int32_t x2 = MultiplyByQuantizedMultiplier(x1, data.effective_scale_2_a, data.effective_scale_2_b); - int32_t x3 = x2 + output_zp; + int32_t x3 = x2 + data.output_zero_point; int32_t x4 = std::min(std::max(output_min, x3), output_max); - GetTensorData(output_tensor)[i] = static_cast(x4); + tflite::micro::GetTensorData(output_tensor)[i] = + static_cast(x4); } } } -} // namespace - -// Input tensors. -constexpr int kInputTensor = 0; -constexpr int kWeightsFeatureTensor = 1; -constexpr int kWeightsTimeTensor = 2; -constexpr int kBiasTensor = 3; -// This is a variable tensor, and will be modified by this op. -constexpr int kInputActivationStateTensor = 4; - -// Output tensor. -constexpr int kOutputTensor = 0; - void* Init(TfLiteContext* context, const char* buffer, size_t length) { TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); - void* data = nullptr; - if (context->AllocatePersistentBuffer(context, sizeof(OpData), &data) == - kTfLiteError) { - return nullptr; - } - return data; + return context->AllocatePersistentBuffer(context, sizeof(OpData)); } TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { @@ -359,13 +362,17 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { // [4] = Activation State (variable), // {2, batch_size, memory_size * num_filters} const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); const TfLiteTensor* weights_feature = GetInput(context, node, kWeightsFeatureTensor); + TF_LITE_ENSURE(context, weights_feature != nullptr); const TfLiteTensor* weights_time = GetInput(context, node, kWeightsTimeTensor); + TF_LITE_ENSURE(context, weights_time != nullptr); const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); const TfLiteTensor* activation_state = GetInput(context, node, kInputActivationStateTensor); + TF_LITE_ENSURE(context, activation_state != nullptr); // Define input constants based on input tensor definition above: const int rank = params->rank; @@ -382,9 +389,10 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, NumDimensions(input), 2); // Validate Tensor Output: - // [0] = float/int8, {2, batch_size, num_units} + // [0] = float/int8_t, {2, batch_size, num_units} TF_LITE_ENSURE_EQ(context, node->outputs->size, 1); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); TF_LITE_ENSURE_EQ(context, NumDimensions(output), 2); TF_LITE_ENSURE_EQ(context, output->dims->data[0], batch_size); TF_LITE_ENSURE_EQ(context, output->dims->data[1], num_units); @@ -408,9 +416,14 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, activation_state->dims->data[0], batch_size); TF_LITE_ENSURE_EQ(context, activation_state->dims->data[1], memory_size * num_filters); + // Since is_variable is not part of TFLiteEvalTensor, check is_variable here. + TF_LITE_ENSURE_EQ(context, activation_state->is_variable, true); TF_LITE_ENSURE_EQ(context, node->inputs->size, 5); + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = static_cast(node->user_data); + if (input->type == kTfLiteInt8) { TF_LITE_ENSURE_EQ(context, weights_feature->type, kTfLiteInt8); TF_LITE_ENSURE_EQ(context, weights_time->type, kTfLiteInt16); @@ -419,35 +432,30 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, bias->type, kTfLiteInt32); } - TF_LITE_ENSURE_EQ(context, output->type, kTfLiteInt8); + TF_LITE_ENSURE_TYPES_EQ(context, output->type, kTfLiteInt8); - const auto* input_params = - reinterpret_cast(input->quantization.params); - const auto* weights_feature_params = - static_cast( - weights_feature->quantization.params); - const auto* state_params = static_cast( - activation_state->quantization.params); - const auto* weight_time_params = - static_cast( - weights_time->quantization.params); - const auto* output_params = static_cast( - output->quantization.params); const double effective_scale_1 = static_cast( - input_params->scale->data[0] * weights_feature_params->scale->data[0] / - state_params->scale->data[0]); - const double effective_scale_2 = static_cast( - state_params->scale->data[0] * weight_time_params->scale->data[0] / - output_params->scale->data[0]); + input->params.scale * weights_feature->params.scale / + activation_state->params.scale); + const double effective_scale_2 = + static_cast(activation_state->params.scale * + weights_time->params.scale / output->params.scale); - TFLITE_DCHECK(node->user_data != nullptr); - OpData* data = static_cast(node->user_data); + // TODO(b/162018098): Use TF_LITE_ENSURE_NEAR when it is ready. + TF_LITE_ENSURE( + context, + std::abs(static_cast(bias->params.scale) - + static_cast(activation_state->params.scale * + weights_time->params.scale)) < 1e-5); QuantizeMultiplier(effective_scale_1, &(data->effective_scale_1_a), &(data->effective_scale_1_b)); QuantizeMultiplier(effective_scale_2, &(data->effective_scale_2_a), &(data->effective_scale_2_b)); + data->input_zero_point = input->params.zero_point; + data->output_zero_point = output->params.zero_point; + TFLITE_DCHECK(context->RequestScratchBufferInArena != nullptr); const TfLiteStatus scratch_status = context->RequestScratchBufferInArena( @@ -467,10 +475,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { if (bias != nullptr) { TF_LITE_ENSURE_EQ(context, bias->type, kTfLiteFloat32); } - TF_LITE_ENSURE_EQ(context, output->type, kTfLiteFloat32); - - TFLITE_DCHECK(node->user_data != nullptr); - OpData* data = static_cast(node->user_data); + TF_LITE_ENSURE_TYPES_EQ(context, output->type, kTfLiteFloat32); TFLITE_DCHECK(context->RequestScratchBufferInArena != nullptr); const TfLiteStatus scratch_status = context->RequestScratchBufferInArena( @@ -484,20 +489,24 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { auto* params = reinterpret_cast(node->builtin_data); - - const TfLiteTensor* input = GetInput(context, node, kInputTensor); - const TfLiteTensor* weights_feature = - GetInput(context, node, kWeightsFeatureTensor); - const TfLiteTensor* weights_time = - GetInput(context, node, kWeightsTimeTensor); - const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); - TfLiteTensor* activation_state = - GetVariableInput(context, node, kInputActivationStateTensor); - TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - TFLITE_DCHECK(node->user_data != nullptr); const OpData& data = *(static_cast(node->user_data)); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + const TfLiteEvalTensor* weights_feature = + tflite::micro::GetEvalInput(context, node, kWeightsFeatureTensor); + const TfLiteEvalTensor* weights_time = + tflite::micro::GetEvalInput(context, node, kWeightsTimeTensor); + const TfLiteEvalTensor* bias = + (NumInputs(node) == 5) + ? tflite::micro::GetEvalInput(context, node, kBiasTensor) + : nullptr; + TfLiteEvalTensor* activation_state = tflite::micro::GetMutableEvalInput( + context, node, kInputActivationStateTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + switch (weights_feature->type) { case kTfLiteFloat32: { EvalFloatSVDF(context, node, input, weights_feature, weights_time, bias, @@ -508,11 +517,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } case kTfLiteInt8: { - TF_LITE_ENSURE_EQ(context, params->activation, kTfLiteActRelu); - EvalIntegerSVDF(context, node, input, weights_feature, weights_time, bias, - params, activation_state, output, data, - input->params.zero_point, output->params.zero_point); + params, activation_state, output, data); return kTfLiteOk; break; } @@ -525,20 +531,17 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } -} // namespace svdf +} // namespace -TfLiteRegistration* Register_SVDF() { - static TfLiteRegistration r = {/*init=*/svdf::Init, - /*free=*/nullptr, - /*prepare=*/svdf::Prepare, - /*invoke=*/svdf::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_SVDF() { + return {/*init=*/Init, + /*free=*/nullptr, + /*prepare=*/Prepare, + /*invoke=*/Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } -} // namespace micro -} // namespace ops } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/tanh.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/tanh.cc new file mode 100644 index 00000000..7743a87f --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/tanh.cc @@ -0,0 +1,158 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h" + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/tanh.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/micro_utils.h" + +namespace tflite { +namespace ops { +namespace micro { +namespace activations { +namespace { +constexpr int kInputTensor = 0; +constexpr int kOutputTensor = 0; + +struct OpData { + int32_t input_zero_point; + int32_t input_range_radius; + int32_t input_multiplier; + int input_left_shift; +}; + +void* TanhInit(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + +TfLiteStatus CalculateArithmeticOpData(TfLiteContext* context, TfLiteNode* node, + OpData* data) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); + + if (input->type == kTfLiteUInt8 || input->type == kTfLiteInt8) { + static constexpr int kInputIntegerBits = 4; + const double input_real_multiplier = + static_cast(input->params.scale) * + static_cast(1 << (31 - kInputIntegerBits)); + + const double q = std::frexp(input_real_multiplier, &data->input_left_shift); + data->input_multiplier = static_cast(TfLiteRound(q * (1ll << 31))); + + data->input_range_radius = + CalculateInputRadius(kInputIntegerBits, data->input_left_shift, 31); + } + return kTfLiteOk; +} + +TfLiteStatus TanhPrepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + + OpData* data = static_cast(node->user_data); + + const TfLiteTensor* input = GetInput(context, node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); + data->input_zero_point = input->params.zero_point; + return CalculateArithmeticOpData(context, node, data); +} + +} // namespace + +TfLiteStatus TanhEval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + TFLITE_DCHECK(node->user_data != nullptr); + const OpData& data = *(static_cast(node->user_data)); + + switch (input->type) { + case kTfLiteFloat32: { + reference_ops::Tanh(tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + return kTfLiteOk; + } break; + case kTfLiteInt16: { + TanhParams params; + params.input_left_shift = data.input_left_shift; + reference_ops::Tanh(params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + return kTfLiteOk; + } break; + case kTfLiteUInt8: { + TanhParams params; + params.input_zero_point = data.input_zero_point; + params.input_range_radius = data.input_range_radius; + params.input_multiplier = data.input_multiplier; + params.input_left_shift = data.input_left_shift; + reference_ops::Tanh(params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + + return kTfLiteOk; + } break; + case kTfLiteInt8: { + reference_integer_ops::Tanh( + data.input_zero_point, data.input_range_radius, data.input_multiplier, + data.input_left_shift, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + return kTfLiteOk; + } break; + default: + TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", + TfLiteTypeGetName(input->type), + TfLiteTypeGetName(output->type)); + return kTfLiteError; + } +} + +} // namespace activations + +TfLiteRegistration Register_TANH() { + return {/*init=*/activations::TanhInit, + /*free=*/nullptr, + /*prepare=*/activations::TanhPrepare, + /*invoke=*/activations::TanhEval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; +} +} // namespace micro +} // namespace ops +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/kernels/unpack.cc b/code/lib/tfmicro/tensorflow/lite/micro/kernels/unpack.cc index faa032d7..557cc57a 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/kernels/unpack.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/kernels/unpack.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" namespace tflite { namespace ops { @@ -28,14 +29,16 @@ constexpr int kInputTensor = 0; template TfLiteStatus UnpackImpl(TfLiteContext* context, TfLiteNode* node, - const TfLiteTensor* input, int output_count, int axis) { - const TfLiteTensor* output0 = GetOutput(context, node, 0); + const TfLiteEvalTensor* input, int output_count, + int axis) { + const TfLiteEvalTensor* output0 = + tflite::micro::GetEvalOutput(context, node, 0); const TfLiteIntArray* input_dims = input->dims; const TfLiteIntArray* output_dims = output0->dims; const int dimensions = input_dims->size; if (axis < 0) { - axis += NumDimensions(input); + axis += input->dims->size; } TFLITE_DCHECK_LT(axis, dimensions); @@ -54,11 +57,11 @@ TfLiteStatus UnpackImpl(TfLiteContext* context, TfLiteNode* node, } TFLITE_DCHECK_EQ(output_size, copy_size * outer_size); - const T* input_data = GetTensorData(input); + const T* input_data = tflite::micro::GetTensorData(input); for (int i = 0; i < output_count; ++i) { - TfLiteTensor* t = GetOutput(context, node, i); - T* output_data = GetTensorData(t); + TfLiteEvalTensor* t = tflite::micro::GetEvalOutput(context, node, i); + T* output_data = tflite::micro::GetTensorData(t); for (int k = 0; k < outer_size; ++k) { T* output_ptr = output_data + copy_size * k; int loc = k * output_count * copy_size + i * copy_size; @@ -74,7 +77,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TfLiteUnpackParams* data = reinterpret_cast(node->builtin_data); - const TfLiteTensor* input = GetInput(context, node, kInputTensor); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); switch (input->type) { case kTfLiteFloat32: { @@ -101,16 +105,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace } // namespace unpack -TfLiteRegistration* Register_UNPACK() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/unpack::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; - return &r; +TfLiteRegistration Register_UNPACK() { + return {/*init=*/nullptr, + /*free=*/nullptr, + /*prepare=*/nullptr, + /*invoke=*/unpack::Eval, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0}; } } // namespace micro diff --git a/code/lib/tfmicro/tensorflow/lite/micro/memory_helpers.cc b/code/lib/tfmicro/tensorflow/lite/micro/memory_helpers.cc index 302f160a..c6180cb4 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/memory_helpers.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/memory_helpers.cc @@ -15,9 +15,15 @@ limitations under the License. #include "tensorflow/lite/micro/memory_helpers.h" +#include #include +#include "flatbuffers/flatbuffers.h" // from @flatbuffers +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/core/api/flatbuffer_conversions.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/schema/schema_generated.h" namespace tflite { @@ -40,8 +46,7 @@ size_t AlignSizeUp(size_t size, size_t alignment) { return aligned_size; } -TfLiteStatus TfLiteTypeSizeOf(TfLiteType type, size_t* size, - ErrorReporter* reporter) { +TfLiteStatus TfLiteTypeSizeOf(TfLiteType type, size_t* size) { switch (type) { case kTfLiteFloat32: *size = sizeof(float); @@ -67,9 +72,10 @@ TfLiteStatus TfLiteTypeSizeOf(TfLiteType type, size_t* size, case kTfLiteComplex64: *size = sizeof(float) * 2; break; + case kTfLiteComplex128: + *size = sizeof(double) * 2; + break; default: - reporter->Report("Type %s (%d) not is not supported", - TfLiteTypeGetName(type), type); return kTfLiteError; } return kTfLiteOk; @@ -79,17 +85,71 @@ TfLiteStatus BytesRequiredForTensor(const tflite::Tensor& flatbuffer_tensor, size_t* bytes, size_t* type_size, ErrorReporter* error_reporter) { int element_count = 1; - for (size_t n = 0; n < flatbuffer_tensor.shape()->Length(); ++n) { - element_count *= flatbuffer_tensor.shape()->Get(n); + // If flatbuffer_tensor.shape == nullptr, then flatbuffer_tensor is a scalar + // so has 1 element. + if (flatbuffer_tensor.shape() != nullptr) { + for (size_t n = 0; n < flatbuffer_tensor.shape()->Length(); ++n) { + element_count *= flatbuffer_tensor.shape()->Get(n); + } } TfLiteType tf_lite_type; TF_LITE_ENSURE_STATUS(ConvertTensorType(flatbuffer_tensor.type(), &tf_lite_type, error_reporter)); - TF_LITE_ENSURE_STATUS( - TfLiteTypeSizeOf(tf_lite_type, type_size, error_reporter)); + TF_LITE_ENSURE_STATUS(TfLiteTypeSizeOf(tf_lite_type, type_size)); *bytes = element_count * (*type_size); return kTfLiteOk; } +TfLiteStatus TfLiteEvalTensorByteLength(const TfLiteEvalTensor* eval_tensor, + size_t* out_bytes) { + TFLITE_DCHECK(out_bytes != nullptr); + + int element_count = 1; + // If eval_tensor->dims == nullptr, then tensor is a scalar so has 1 element. + if (eval_tensor->dims != nullptr) { + for (int n = 0; n < eval_tensor->dims->size; ++n) { + element_count *= eval_tensor->dims->data[n]; + } + } + size_t type_size; + TF_LITE_ENSURE_STATUS(TfLiteTypeSizeOf(eval_tensor->type, &type_size)); + *out_bytes = element_count * type_size; + return kTfLiteOk; +} + +TfLiteStatus AllocateOutputDimensionsFromInput(TfLiteContext* context, + const TfLiteTensor* input1, + const TfLiteTensor* input2, + TfLiteTensor* output) { + const TfLiteTensor* input = nullptr; + + TF_LITE_ENSURE(context, input1->dims != nullptr); + TF_LITE_ENSURE(context, input2->dims != nullptr); + TF_LITE_ENSURE(context, output->dims->size == 0); + + input = input1->dims->size > input2->dims->size ? input1 : input2; + TF_LITE_ENSURE(context, output->type == input->type); + + size_t size = 0; + TfLiteTypeSizeOf(input->type, &size); + const int dimensions_count = tflite::GetTensorShape(input).DimensionsCount(); + for (int i = 0; i < dimensions_count; i++) { + size *= input->dims->data[i]; + } + + output->bytes = size; + + output->dims = + reinterpret_cast(context->AllocatePersistentBuffer( + context, TfLiteIntArrayGetSizeInBytes(size))); + + output->dims->size = input->dims->size; + for (int i = 0; i < dimensions_count; i++) { + output->dims->data[i] = input->dims->data[i]; + } + + return kTfLiteOk; +} + } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/memory_helpers.h b/code/lib/tfmicro/tensorflow/lite/micro/memory_helpers.h index ef8205c8..8f5526ce 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/memory_helpers.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/memory_helpers.h @@ -15,6 +15,9 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_MEMORY_HELPERS_H_ #define TENSORFLOW_LITE_MICRO_MEMORY_HELPERS_H_ +#include +#include + #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/schema/schema_generated.h" @@ -31,14 +34,26 @@ uint8_t* AlignPointerDown(uint8_t* data, size_t alignment); size_t AlignSizeUp(size_t size, size_t alignment); // Returns size in bytes for a given TfLiteType. -TfLiteStatus TfLiteTypeSizeOf(TfLiteType type, size_t* size, - ErrorReporter* reporter); +TfLiteStatus TfLiteTypeSizeOf(TfLiteType type, size_t* size); // How many bytes are needed to hold a tensor's contents. TfLiteStatus BytesRequiredForTensor(const tflite::Tensor& flatbuffer_tensor, size_t* bytes, size_t* type_size, ErrorReporter* error_reporter); +// How many bytes are used in a TfLiteEvalTensor instance. The byte length is +// returned in out_bytes. +TfLiteStatus TfLiteEvalTensorByteLength(const TfLiteEvalTensor* eval_tensor, + size_t* out_bytes); + +// Deduce output dimensions from input and allocate given size. +// Useful for operators with two inputs where the largest input should equal the +// output dimension. +TfLiteStatus AllocateOutputDimensionsFromInput(TfLiteContext* context, + const TfLiteTensor* input1, + const TfLiteTensor* input2, + TfLiteTensor* output); + } // namespace tflite #endif // TENSORFLOW_LITE_MICRO_MEMORY_HELPERS_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc b/code/lib/tfmicro/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc index c5e2d579..39991ab7 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc @@ -48,10 +48,10 @@ GreedyMemoryPlanner::GreedyMemoryPlanner(unsigned char* scratch_buffer, requirements_ = reinterpret_cast(next_free); next_free += sizeof(BufferRequirements) * max_buffer_count_; - buffer_sizes_sorted_by_size_ = reinterpret_cast(next_free); + buffer_sizes_sorted_ = reinterpret_cast(next_free); next_free += sizeof(int) * max_buffer_count_; - buffer_ids_sorted_by_size_ = reinterpret_cast(next_free); + buffer_ids_sorted_ = reinterpret_cast(next_free); next_free += sizeof(int) * max_buffer_count_; buffers_sorted_by_offset_ = reinterpret_cast(next_free); @@ -76,11 +76,24 @@ TfLiteStatus GreedyMemoryPlanner::AddBuffer( current->size = size; current->first_time_used = first_time_used; current->last_time_used = last_time_used; + current->offline_offset = kOnlinePlannedBuffer; ++buffer_count_; need_to_calculate_offsets_ = true; return kTfLiteOk; } +TfLiteStatus GreedyMemoryPlanner::AddBuffer( + tflite::ErrorReporter* error_reporter, int size, int first_time_used, + int last_time_used, int offline_offset) { + BufferRequirements* current = &requirements_[buffer_count_]; + if (AddBuffer(error_reporter, size, first_time_used, last_time_used) != + kTfLiteOk) { + return kTfLiteError; + } + current->offline_offset = offline_offset; + return kTfLiteOk; +} + bool GreedyMemoryPlanner::DoesEntryOverlapInTime( const GreedyMemoryPlanner::ListEntry* entry, const int first_time_used, const int last_time_used) const { @@ -102,7 +115,7 @@ GreedyMemoryPlanner::NextSimultaneouslyActiveBuffer( ListEntry* result = nullptr; ListEntry* candidate_next_entry; if (start == nullptr) { - candidate_next_entry = &buffers_sorted_by_offset_[0]; + candidate_next_entry = &buffers_sorted_by_offset_[first_entry_index_]; } else { if (start->next_entry_index == -1) { return nullptr; @@ -134,29 +147,51 @@ void GreedyMemoryPlanner::CalculateOffsetsIfNeeded() { // This helps find a more compact layout. Intuitively, you can think // about putting the large buffers in place first, and then the // smaller buffers can fit in the gaps, rather than fragmenting the - // gaps with small buffers at the beginning. + // gaps with small buffers at the beginning. Add offline planned offsets + // first in the list, since they have a predetermined offset. + int idx_from_tail = buffer_count_; + int idx_from_head = 0; for (int i = 0; i < buffer_count_; ++i) { - buffer_sizes_sorted_by_size_[i] = requirements_[i].size; - buffer_ids_sorted_by_size_[i] = i; - buffer_offsets_[i] = -1; + if (requirements_[i].offline_offset == kOnlinePlannedBuffer) { + idx_from_tail--; + buffer_sizes_sorted_[idx_from_tail] = requirements_[i].size; + buffer_ids_sorted_[idx_from_tail] = i; + buffer_offsets_[i] = -1; + } else { + buffer_sizes_sorted_[idx_from_head] = requirements_[i].size; + buffer_ids_sorted_[idx_from_head] = i; + buffer_offsets_[i] = requirements_[i].offline_offset; + idx_from_head++; + } } - // This sorting algorithm is naive, and may end up taking a very long time - // with hundreds of buffers. - ReverseSortInPlace(buffer_sizes_sorted_by_size_, buffer_ids_sorted_by_size_, - buffer_count_); - // Put the largest buffer at offset zero to start the process. - ListEntry* first_entry = &buffers_sorted_by_offset_[0]; - first_entry->offset = 0; - first_entry->requirements_index = buffer_ids_sorted_by_size_[0]; - first_entry->next_entry_index = -1; + // This sorting algorithm is naive, and may end up taking a very long time + // with hundreds of buffers. Do not sort the offline planned offsets. + ReverseSortInPlace(&buffer_sizes_sorted_[idx_from_head], + &buffer_ids_sorted_[idx_from_head], + buffer_count_ - idx_from_head); + + // Initialize the first entry to the first buffer in + // buffer_ids_sorted_. + // - If there are no offline planned offsets, the largest buffer will be + // first, and the buffers will be handled in size order. + // - If offline offsets are present, these will be handled first in order + // for the greedy algorithm to utilized gaps in the offline plan. + first_entry_index_ = 0; next_free_entry_ = 1; - buffer_offsets_[buffer_ids_sorted_by_size_[0]] = 0; + ListEntry* first_entry = &buffers_sorted_by_offset_[first_entry_index_]; + first_entry->next_entry_index = -1; // to mark the entry as end of list + int buffer_id = buffer_ids_sorted_[0]; + first_entry->requirements_index = buffer_id; + if (requirements_[buffer_id].offline_offset == kOnlinePlannedBuffer) { + buffer_offsets_[buffer_id] = 0; + } + first_entry->offset = buffer_offsets_[buffer_id]; // Work through the rest of the buffers to find a good gap to place each one. for (int i = 1; i < buffer_count_; ++i) { // The id is the order the buffer was originally added by the client. - const int buffer_id = buffer_ids_sorted_by_size_[i]; + buffer_id = buffer_ids_sorted_[i]; // Look at what size and time range the buffer needs to be active. BufferRequirements* wanted_requirements = &requirements_[buffer_id]; const int wanted_size = wanted_requirements->size; @@ -168,37 +203,43 @@ void GreedyMemoryPlanner::CalculateOffsetsIfNeeded() { // so that it's easy to find the next buffer in memory, and so the gap. // The candidate_entry variable holds the buffer that we're considering // placing the current buffer after. - ListEntry* prior_entry = nullptr; + int candidate_offset = 0; // Loop through the offset-ordered list of buffers, looking for gaps. - while (true) { - // Find out what the next active buffer is. - ListEntry* next_entry = NextSimultaneouslyActiveBuffer( - prior_entry, wanted_first_time_used, wanted_last_time_used); + if (wanted_requirements->offline_offset == kOnlinePlannedBuffer) { + ListEntry* prior_entry = nullptr; + while (true) { + // Find out what the next active buffer is. + ListEntry* next_entry = NextSimultaneouslyActiveBuffer( + prior_entry, wanted_first_time_used, wanted_last_time_used); - if (prior_entry) { - BufferRequirements* candidate_requirements = - &requirements_[prior_entry->requirements_index]; - const int prior_entry_offset = - prior_entry->offset + candidate_requirements->size; - if (prior_entry_offset > candidate_offset) { - candidate_offset = prior_entry_offset; + if (prior_entry) { + BufferRequirements* candidate_requirements = + &requirements_[prior_entry->requirements_index]; + const int prior_entry_offset = + prior_entry->offset + candidate_requirements->size; + if (prior_entry_offset > candidate_offset) { + candidate_offset = prior_entry_offset; + } } + if (next_entry == nullptr) { + // We're at the end of the list, so we can always append the buffer + // here. + break; + } + // Find out how much space there is between us and the next buffer. + const int gap = next_entry->offset - candidate_offset; + if (gap >= wanted_size) { + // This entry has a big enough gap between it and the next, so + // use it! + break; + } + // The gap wasn't big enough, so move on to another candidate. + prior_entry = next_entry; } - if (next_entry == nullptr) { - // We're at the end of the list, so we can always append the buffer - // here. - break; - } - // Find out how much space there is between us and the next buffer. - const int gap = next_entry->offset - candidate_offset; - if (gap >= wanted_size) { - // This entry has a big enough gap between it and the next, so - // use it! - break; - } - // The gap wasn't big enough, so move on to another candidate. - prior_entry = next_entry; + } else { + // Offline planned offset are to be considered constant + candidate_offset = wanted_requirements->offline_offset; } // At this point, we've either found a gap (possibly at the end of the // list) and want to place the buffer there, or there are no other active @@ -212,26 +253,36 @@ void GreedyMemoryPlanner::CalculateOffsetsIfNeeded() { new_entry->requirements_index = buffer_id; const int new_entry_index = next_free_entry_; ++next_free_entry_; - ListEntry* current_entry = first_entry; - // Make sure that we insert the buffer at the correct place in the ordered - // list. - while (true) { - const int next_entry_index = current_entry->next_entry_index; - if (next_entry_index == -1) { - // We're at the end of the list, so just add the new entry here. - current_entry->next_entry_index = new_entry_index; - new_entry->next_entry_index = -1; - break; + + if (first_entry->offset > candidate_offset) { + // The new entry offset is smaller than the first entry offset => + // replace the first entry + first_entry = new_entry; + first_entry->next_entry_index = first_entry_index_; + first_entry_index_ = new_entry_index; + } else { + ListEntry* current_entry = first_entry; + // Make sure that we insert the buffer at the correct place in the + // buffer-offset-ordered list + while (true) { + const int next_entry_index = current_entry->next_entry_index; + if (next_entry_index == -1) { + // We're at the end of the list, so just add the new entry here. + current_entry->next_entry_index = new_entry_index; + new_entry->next_entry_index = -1; + break; + } + // not at the end of the list -> take a look at next entry + ListEntry* next_entry = &buffers_sorted_by_offset_[next_entry_index]; + if (next_entry->offset > candidate_offset) { + // We're at the right spot to do an insertion and retain the sorting + // order, so place the new entry here. + new_entry->next_entry_index = current_entry->next_entry_index; + current_entry->next_entry_index = new_entry_index; + break; + } + current_entry = next_entry; } - ListEntry* next_entry = &buffers_sorted_by_offset_[next_entry_index]; - if (next_entry->offset > candidate_offset) { - // We're at the right spot to do an insertion and retain the sorting - // order, so place the new entry here. - new_entry->next_entry_index = current_entry->next_entry_index; - current_entry->next_entry_index = new_entry_index; - break; - } - current_entry = next_entry; } } } @@ -241,7 +292,7 @@ size_t GreedyMemoryPlanner::GetMaximumMemorySize() { if (buffer_count_ == 0) { return 0; } - ListEntry* entry = &buffers_sorted_by_offset_[0]; + ListEntry* entry = &buffers_sorted_by_offset_[first_entry_index_]; size_t max_size = 0; while (entry) { BufferRequirements* requirements = diff --git a/code/lib/tfmicro/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h b/code/lib/tfmicro/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h index 0cb81093..f5f26a8b 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h @@ -21,6 +21,8 @@ limitations under the License. namespace tflite { +constexpr int kOnlinePlannedBuffer = -1; + // A memory planner that uses a greedy algorithm to arrange buffers in memory // to minimize the overall arena size needed. // @@ -59,6 +61,12 @@ class GreedyMemoryPlanner : public MemoryPlanner { TfLiteStatus AddBuffer(ErrorReporter* error_reporter, int size, int first_time_used, int last_time_used) override; + // Record details of an offline planned buffer offset we want to place. + // offline_offset is the buffer offset from the start of the arena. + TfLiteStatus AddBuffer(ErrorReporter* error_reporter, int size, + int first_time_used, int last_time_used, + int offline_offset); + // Returns the high-water mark of used memory. This is the minimum size of a // memory arena you'd need to allocate to hold these buffers. size_t GetMaximumMemorySize() override; @@ -90,8 +98,8 @@ class GreedyMemoryPlanner : public MemoryPlanner { static size_t per_buffer_size() { const int per_buffer_size = sizeof(BufferRequirements) + // requirements_ - sizeof(int) + // buffer_sizes_sorted_by_size_ - sizeof(int) + // buffer_ids_sorted_by_size_ + sizeof(int) + // buffer_sizes_sorted_ + sizeof(int) + // buffer_ids_sorted_ sizeof(ListEntry) + // buffers_sorted_by_offset_ sizeof(int); // buffer_offsets_; return per_buffer_size; @@ -121,16 +129,25 @@ class GreedyMemoryPlanner : public MemoryPlanner { // Records the client-provided information about each buffer. struct BufferRequirements { int size; + int offline_offset; int first_time_used; int last_time_used; }; // Working arrays used during the layout algorithm. BufferRequirements* requirements_; - int* buffer_sizes_sorted_by_size_; - int* buffer_ids_sorted_by_size_; + // buffer_sizes_sorted_ and buffer_ids_sorted_ are sorted according to: + // { + // offline planned buffers, + // online planned buffers sorted by size + // } + int* buffer_sizes_sorted_; + int* buffer_ids_sorted_; ListEntry* buffers_sorted_by_offset_; - int next_free_entry_; + int next_free_entry_; // Index of the next free entry of + // buffers_sorted_by_offset_ + int first_entry_index_; // Index of the first entry (smallest offset) of + // buffers_sorted_by_offset_ // Stores the outcome of the plan, the location of each buffer in the arena. int* buffer_offsets_; diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_allocator.cc b/code/lib/tfmicro/tensorflow/lite/micro/micro_allocator.cc index 28de77cd..675a64d2 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_allocator.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_allocator.cc @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,31 +18,49 @@ limitations under the License. #include #include +#include "flatbuffers/flatbuffers.h" // from @flatbuffers #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/core/api/flatbuffer_conversions.h" #include "tensorflow/lite/core/api/op_resolver.h" #include "tensorflow/lite/core/api/tensor_utils.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/memory_helpers.h" #include "tensorflow/lite/micro/memory_planner/greedy_memory_planner.h" +#include "tensorflow/lite/micro/memory_planner/memory_planner.h" +#include "tensorflow/lite/micro/micro_op_resolver.h" #include "tensorflow/lite/micro/simple_memory_allocator.h" +#include "tensorflow/lite/schema/schema_generated.h" +#include "tensorflow/lite/schema/schema_utils.h" namespace tflite { namespace { + +// Maximum number of scratch buffer requests per operator. Operator kernels that +// request more than this value will receive an exception. +constexpr size_t kMaxScratchBuffersPerOp = 8; + +// Sentinel value used as a placeholder to mark a ScratchBufferRequest request +// needs a node id assignment. +constexpr int kUnassignedScratchBufferRequestIndex = -1; + // Used to hold information used during allocation calculations. struct AllocationInfo { size_t bytes; + void** output_ptr; int first_created; int last_used; + int32_t offline_offset; bool needs_allocating; - void** output_ptr; }; // We align tensor buffers to 16-byte boundaries, since this is a common // requirement for SIMD extensions. constexpr int kBufferAlignment = 16; +constexpr char kOfflineMemAllocMetadata[] = "OfflineMemoryAllocation"; +const TfLiteIntArray kZeroLengthIntArray = {}; class MicroBuiltinDataAllocator : public BuiltinDataAllocator { public: @@ -63,87 +81,134 @@ class MicroBuiltinDataAllocator : public BuiltinDataAllocator { TF_LITE_REMOVE_VIRTUAL_DELETE }; -TfLiteStatus AllocateVariables( - const flatbuffers::Vector>* flatbuffer_tensors, - TfLiteTensor* runtime_tensors, SimpleMemoryAllocator* allocator) { - for (size_t i = 0; i < flatbuffer_tensors->size(); ++i) { - if (flatbuffer_tensors->Get(i)->is_variable()) { - runtime_tensors[i].data.data = allocator->AllocateFromTail( - runtime_tensors[i].bytes, kBufferAlignment); - // Allocation failure. - if (runtime_tensors[i].data.data == nullptr) { - return kTfLiteError; +#if !defined(__clang__) +// Helper function to check flatbuffer metadata correctness. This function is +// not called by default. Hence it's not linked in to the final binary code. +TfLiteStatus CheckOfflinePlannedOffsets(const Model* model, + ErrorReporter* error_reporter) { + // Suppress compile warning for unused function + (void)CheckOfflinePlannedOffsets; + + if (model->metadata()) { + for (size_t i = 0; i < model->metadata()->size(); ++i) { + auto metadata = model->metadata()->Get(i); + if (strncmp(metadata->name()->c_str(), kOfflineMemAllocMetadata, + strlen(kOfflineMemAllocMetadata)) == 0) { + auto* subgraphs = model->subgraphs(); + const SubGraph* subgraph = (*subgraphs)[0]; + const flatbuffers::Vector>* tensors = + subgraph->tensors(); + const flatbuffers::Vector>* buffers = + model->buffers(); + int nbr_tflite_tensors = tensors->size(); + auto* buffer = (*buffers)[metadata->buffer()]; + auto* array = buffer->data(); + const uint32_t* metadata_buffer = (uint32_t*)array->data(); + int version = metadata_buffer[0]; + int subgraph_idx = metadata_buffer[1]; + const int nbr_offline_offsets = metadata_buffer[2]; +#ifndef TF_LITE_STRIP_ERROR_STRINGS + int* offline_planner_offsets = (int*)&metadata_buffer[3]; +#endif + + TF_LITE_REPORT_ERROR(error_reporter, "==== Model metadata info: ====="); + TF_LITE_REPORT_ERROR(error_reporter, + "Offline planner metadata found, version %d, " + "subgraph %d, nbr offline offsets %d", + version, subgraph_idx, nbr_offline_offsets); + for (int j = 0; j < nbr_offline_offsets; ++j) { + TF_LITE_REPORT_ERROR( + error_reporter, + "Offline planner tensor index %d, offline offset: %d", j, + offline_planner_offsets[j]); + } + + if (version != 1) { + TF_LITE_REPORT_ERROR(error_reporter, "Version not supported! (%d)\n", + version); + return kTfLiteError; + } + if (subgraph_idx != 0) { + TF_LITE_REPORT_ERROR(error_reporter, + "Only 1 subgraph supported! Subgraph idx (%d)\n", + subgraph_idx); + return kTfLiteError; + } + if (nbr_tflite_tensors != nbr_offline_offsets) { + TF_LITE_REPORT_ERROR(error_reporter, + "Nbr of offline buffer offsets (%d) in metadata " + "not equal nbr tensors (%d)\n", + nbr_offline_offsets, nbr_tflite_tensors); + return kTfLiteError; + } } } - tflite::ResetVariableTensor(&(runtime_tensors[i])); } return kTfLiteOk; } +#endif // A helper class to construct AllocationInfo array. This array contains the // lifetime of tensors / scratch_buffer and will be used to calculate the memory // plan. Methods need to be called in order from `Init`, `Add*`, to `Finish`. class AllocationInfoBuilder { public: - AllocationInfoBuilder(ErrorReporter* reporter, - SimpleMemoryAllocator* allocator) - : reporter_(reporter), allocator_(allocator) {} + AllocationInfoBuilder(AllocationInfo* info, size_t tensor_count, + size_t scratch_buffer_count, ErrorReporter* reporter) + : info_(info), + tensor_count_(tensor_count), + buffer_count_(scratch_buffer_count), + reporter_(reporter) {} - // Initializes the builder by allocating AllocationInfo array from the - // simple memory allocator. - TfLiteStatus Init(size_t tensor_count, size_t scratch_buffer_count) { - tensor_count_ = tensor_count; - buffer_count_ = scratch_buffer_count; - return Allocate(); - } + // Check if model contains offline planned buffer offsets. + // - If there's no metadata available, offline_planner_offsets is not set + // - If there's metadata available, offline_planner_offsets will point to the + // first offset in the metadata buffer list. + TfLiteStatus GetOfflinePlannedOffsets( + const Model* model, const int32_t** offline_planner_offsets); // Add allocaiton information for the tensors. TfLiteStatus AddTensors(const SubGraph* subgraph, - TfLiteTensor* runtime_tensors); + const int32_t* offline_offsets, + TfLiteEvalTensor* eval_tensors); + // Add allocation information for the scratch buffers. - TfLiteStatus AddScratchBuffers(internal::ScratchBufferHandle* buffer_handles); + TfLiteStatus AddScratchBuffers( + internal::ScratchBufferRequest* scratch_buffer_requests, + ScratchBufferHandle* scratch_buffer_handles); // Returns a pointer to the built AllocationInfo array. const AllocationInfo* Finish() const { return info_; } - size_t Size() const { return tensor_count_ + buffer_count_; } private: - // Allocate the output AllocationInfo array from the allocator_; - TfLiteStatus Allocate(); - - ErrorReporter* reporter_ = nullptr; - SimpleMemoryAllocator* allocator_ = nullptr; + AllocationInfo* info_ = nullptr; size_t tensor_count_ = 0; size_t buffer_count_ = 0; - AllocationInfo* info_ = nullptr; + ErrorReporter* reporter_ = nullptr; }; -TfLiteStatus AllocationInfoBuilder::Allocate() { - size_t bytes = sizeof(AllocationInfo) * Size(); - info_ = reinterpret_cast( - allocator_->AllocateFromTail(bytes, alignof(AllocationInfo))); - if (info_ == nullptr) { - TF_LITE_REPORT_ERROR( - reporter_, - "Failed to allocate memory for allocation_info, %d bytes required", - bytes); - return kTfLiteError; - } - return kTfLiteOk; -} - TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, - TfLiteTensor* runtime_tensors) { + const int32_t* offline_offsets, + TfLiteEvalTensor* eval_tensors) { + TFLITE_DCHECK(eval_tensors != nullptr); + // Set up allocation info for all tensors. for (size_t i = 0; i < tensor_count_; ++i) { AllocationInfo* current = &info_[i]; - // TfLiteTensor.uint8 field is deprecated so use .data field instead. - current->output_ptr = &(runtime_tensors[i].data.data); - current->bytes = runtime_tensors[i].bytes; + current->output_ptr = &(eval_tensors[i].data.data); + + TF_LITE_ENSURE_STATUS( + TfLiteEvalTensorByteLength(&eval_tensors[i], ¤t->bytes)); + current->first_created = -1; current->last_used = -1; - current->needs_allocating = (runtime_tensors[i].data.data == nullptr) && + current->needs_allocating = (eval_tensors[i].data.data == nullptr) && (!subgraph->tensors()->Get(i)->is_variable()); + if (offline_offsets) { + current->offline_offset = offline_offsets[i]; + } else { + current->offline_offset = kOnlinePlannedBuffer; + } } for (size_t i = 0; i < subgraph->inputs()->size(); ++i) { @@ -178,16 +243,15 @@ TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, } } - // Work out which tensors need to be allocated. + // Sanity check for valid tensor lifetime. for (size_t i = 0; i < tensor_count_; ++i) { AllocationInfo* current = &info_[i]; - const bool is_read_only = + // Even though tensor appears to be read only it may still need to be + // allocated. + const bool appears_read_only = (current->first_created == -1) && (current->last_used != -1); - if (is_read_only) { - current->needs_allocating = false; - } const bool has_partial_lifetime = - !is_read_only && + !appears_read_only && ((current->first_created == -1) || (current->last_used == -1)); if (has_partial_lifetime && current->needs_allocating) { TF_LITE_REPORT_ERROR( @@ -201,23 +265,77 @@ TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, return kTfLiteOk; } +// The tensor offsets will be encoded in the metadata:[Metadata] field of the +// Model. The following encoding applies: +// +// | Metadata component | Value | +// | name:string | “OfflineMemoryAllocation” | +// | buffer:unit | Index of buffer containing memory allocation data | +// +// The buffer contents for the memory allocation is a list of 32-bit integers. +// The number of tensors, n, must be equal to the number of tensors defined in +// the model. The following encoding applies: +// +// | Offset | Value | +// | 0 | Offline allocation format version – set to 0 | +// | 1 | Subgraph index to which this allocation applies | +// | 2 | Number offsets following: n | +// | 3 | Arena byte offset of tensor #0 or -1 to allocate at runtime | +// | 4 | Arena byte offset of tensor #1 or -1 to allocate at runtime | +// | 3+(n-1) | Arena byte offset of tensor #(n-1) or -1 to allocate at runtime | +TfLiteStatus AllocationInfoBuilder::GetOfflinePlannedOffsets( + const Model* model, const int32_t** offline_planner_offsets) { + if (model->metadata()) { + for (size_t i = 0; i < model->metadata()->size(); ++i) { + auto metadata = model->metadata()->Get(i); + if (strncmp(metadata->name()->c_str(), kOfflineMemAllocMetadata, + strlen(kOfflineMemAllocMetadata)) == 0) { + const flatbuffers::Vector>* buffers = + model->buffers(); + auto* buffer = (*buffers)[metadata->buffer()]; + auto* array = buffer->data(); + const uint32_t* metadata_buffer = + reinterpret_cast(array->data()); + const size_t nbr_tensors = static_cast(metadata_buffer[2]); + *offline_planner_offsets = + reinterpret_cast(&metadata_buffer[3]); + + if (tensor_count_ != nbr_tensors) { + TF_LITE_REPORT_ERROR(reporter_, + "Nbr of offline buffer offsets (%d) in metadata " + "not equal nbr tensors (%d)\n", + nbr_tensors, tensor_count_); + return kTfLiteError; + } + } + } + } + return kTfLiteOk; +} + TfLiteStatus AllocationInfoBuilder::AddScratchBuffers( - internal::ScratchBufferHandle* buffer_handles) { + internal::ScratchBufferRequest* scratch_buffer_requests, + ScratchBufferHandle* scratch_buffer_handles) { // Set up allocation info for buffers. for (size_t i = tensor_count_; i < tensor_count_ + buffer_count_; ++i) { + internal::ScratchBufferRequest* current_request = + &(scratch_buffer_requests[i - tensor_count_]); + ScratchBufferHandle* current_handle = + &(scratch_buffer_handles[i - tensor_count_]); + AllocationInfo* current = &info_[i]; - internal::ScratchBufferHandle* handle = - &(buffer_handles[i - tensor_count_]); - current->output_ptr = reinterpret_cast(&handle->data); - current->bytes = handle->bytes; - current->first_created = handle->node_idx; - current->last_used = handle->node_idx; + current->output_ptr = reinterpret_cast(¤t_handle->data); + current->bytes = current_request->bytes; + current->first_created = current_request->node_idx; + current->last_used = current_request->node_idx; + current->offline_offset = kOnlinePlannedBuffer; current->needs_allocating = true; } return kTfLiteOk; } -TfLiteStatus CreatePlan(ErrorReporter* error_reporter, MemoryPlanner* planner, +TfLiteStatus CreatePlan(ErrorReporter* error_reporter, + GreedyMemoryPlanner* planner, const AllocationInfo* allocation_info, size_t allocation_info_size) { // Add the tensors to our allocation plan. @@ -226,9 +344,15 @@ TfLiteStatus CreatePlan(ErrorReporter* error_reporter, MemoryPlanner* planner, if (current->needs_allocating) { size_t aligned_bytes_required = AlignSizeUp(current->bytes, kBufferAlignment); - TF_LITE_ENSURE_STATUS( - planner->AddBuffer(error_reporter, aligned_bytes_required, - current->first_created, current->last_used)); + if (current->offline_offset == kOnlinePlannedBuffer) { + TF_LITE_ENSURE_STATUS( + planner->AddBuffer(error_reporter, aligned_bytes_required, + current->first_created, current->last_used)); + } else { + TF_LITE_ENSURE_STATUS(planner->AddBuffer( + error_reporter, aligned_bytes_required, current->first_created, + current->last_used, current->offline_offset)); + } } } return kTfLiteOk; @@ -256,24 +380,63 @@ TfLiteStatus CommitPlan(ErrorReporter* error_reporter, MemoryPlanner* planner, namespace internal { -TfLiteStatus InitializeRuntimeTensor( - SimpleMemoryAllocator* allocator, const tflite::Tensor& flatbuffer_tensor, - const flatbuffers::Vector>* buffers, - ErrorReporter* error_reporter, TfLiteTensor* result) { - *result = {}; - // Make sure the serialized type is one we know how to deal with, and convert - // it from a flatbuffer enum into a constant used by the kernel C API. - TF_LITE_ENSURE_STATUS(ConvertTensorType(flatbuffer_tensor.type(), - &result->type, error_reporter)); - // Make sure we remember if the serialized tensor is designated as a variable. - result->is_variable = flatbuffer_tensor.is_variable(); +// Handles architecture safe mapping of flatbuffer vectors to a TfLite*Array +// struct. Matching types are required (e.g. float and TfLiteFloatArray). +// Big-endian systems will always allocate dimension array data in the tail +// (persistent) section. +template +TfLiteStatus FlatBufferVectorToTfLiteTypeArray( + SimpleMemoryAllocator* allocator, ErrorReporter* error_reporter, + const flatbuffers::Vector* flatbuffer_array, + kTfLiteArrayType** result) { + TFLITE_DCHECK(error_reporter != nullptr); + TFLITE_DCHECK(flatbuffer_array != nullptr); + // TODO(b/159668691): Consider adding type assertion or breaking this function + // into multiple functions for each type. std::is_same is c++11 and has a + // special updated constructor in c++17 that requires a string argument. + if (FLATBUFFERS_LITTLEENDIAN) { + // On little-endian machines, TfLite*Array happens to have the same memory + // layout as flatbuffers:Vector, so we can + // reinterpret_cast the flatbuffer vector and avoid a copy and malloc. + *result = const_cast( + reinterpret_cast(flatbuffer_array)); + } else { + // Big-endian architecture can not use the same memory layout as + // flatbuffers::Vector. Allocate from the tail and + // copy values from the flatbuffer into the newly allocated chunk. + kTfLiteArrayType* array = + reinterpret_cast(allocator->AllocateFromTail( + TfLiteIntArrayGetSizeInBytes(flatbuffer_array->Length()), + alignof(kTfLiteArrayType))); + if (array == nullptr) { + TF_LITE_REPORT_ERROR( + error_reporter, + "Failed to allocate %d bytes of memory to copy an array.", + TfLiteIntArrayGetSizeInBytes(flatbuffer_array->Length())); + return kTfLiteError; + } + array->size = flatbuffer_array->Length(); + for (int i = 0; i < array->size; ++i) { + array->data[i] = flatbuffer_array->Get(i); + } + *result = array; + } + return kTfLiteOk; +} +// Returns a pointer to any buffer associated with the flatbuffer tensor. Can +// return nullptr if no buffer is found. +void* GetFlatbufferTensorBuffer( + const tflite::Tensor& flatbuffer_tensor, + const flatbuffers::Vector>* buffers) { // We need to figure out where the actual contents of this tensor are stored // in memory. We'll check to see if there's a serialized buffer (pretty much // the same as a constant op in TensorFlow) associated with this tensor first, // and if there is update the runtime structure to point to its location in // memory. // First see if there's any buffer information in the serialized tensor. + // TODO(b/170379532): Add better unit tests to validate flatbuffer values. + void* out_buffer = nullptr; if (auto* buffer = (*buffers)[flatbuffer_tensor.buffer()]) { // If we've found a buffer, does it have any data? if (auto* array = buffer->data()) { @@ -281,10 +444,7 @@ TfLiteStatus InitializeRuntimeTensor( if (array->size()) { // We've found a buffer with valid data, so update the runtime tensor // data structure to point to it. - result->data.data = - const_cast(static_cast(array->data())); - // We set the data from a serialized buffer, so record tha. - result->allocation_type = kTfLiteMmapRo; + out_buffer = const_cast(static_cast(array->data())); } } // TODO(petewarden): It's not clear in what circumstances we could have a @@ -293,6 +453,25 @@ TfLiteStatus InitializeRuntimeTensor( // error condition? It would be good to tighten up the specification to make // it less ambiguous. } + return out_buffer; +} + +TfLiteStatus InitializeTfLiteTensorFromFlatbuffer( + SimpleMemoryAllocator* allocator, bool allocate_temp, + const tflite::Tensor& flatbuffer_tensor, + const flatbuffers::Vector>* buffers, + ErrorReporter* error_reporter, TfLiteTensor* result) { + TFLITE_DCHECK(result != nullptr); + + *result = {}; + // Make sure the serialized type is one we know how to deal with, and convert + // it from a flatbuffer enum into a constant used by the kernel C API. + TF_LITE_ENSURE_STATUS(ConvertTensorType(flatbuffer_tensor.type(), + &result->type, error_reporter)); + // Make sure we remember if the serialized tensor is designated as a variable. + result->is_variable = flatbuffer_tensor.is_variable(); + + result->data.data = GetFlatbufferTensorBuffer(flatbuffer_tensor, buffers); // TODO(petewarden): Some of these paths aren't getting enough testing // coverage, so we should figure out some tests that exercise them. @@ -301,6 +480,9 @@ TfLiteStatus InitializeRuntimeTensor( // make a note that they will be allocated from memory. The actual // allocation won't happen until later. result->allocation_type = kTfLiteArenaRw; + } else { + // We set the data from a serialized buffer, so record tha. + result->allocation_type = kTfLiteMmapRo; } // Figure out what the size in bytes of the buffer is and store it. @@ -308,12 +490,18 @@ TfLiteStatus InitializeRuntimeTensor( TF_LITE_ENSURE_STATUS(BytesRequiredForTensor( flatbuffer_tensor, &result->bytes, &type_size, error_reporter)); - // TFLM doesn't allow reshaping the tensor which requires dynamic memory - // allocation so it is safe to drop the const qualifier. In the future, if we - // really want to update the tensor shape, we can always pass in a new - // TfLiteIntArray - especially we have to do so if the dimension is changed. - result->dims = const_cast( - reinterpret_cast(flatbuffer_tensor.shape())); + if (flatbuffer_tensor.shape() == nullptr) { + // flatbuffer_tensor.shape() can return a nullptr in the case of a scalar + // tensor. + result->dims = const_cast(&kZeroLengthIntArray); + } else { + // TFLM doesn't allow reshaping the tensor which requires dynamic memory + // allocation so it is safe to drop the const qualifier. In the future, if + // we really want to update the tensor shape, we can always pass in a new + // TfLiteIntArray - especially we have to do so if the dimension is + TF_LITE_ENSURE_STATUS(FlatBufferVectorToTfLiteTypeArray( + allocator, error_reporter, flatbuffer_tensor.shape(), &(result->dims))); + } // Copy the quantization information from the serialized data. const auto* src_quantization = flatbuffer_tensor.quantization(); @@ -333,39 +521,45 @@ TfLiteStatus InitializeRuntimeTensor( // Populate per-channel quantization params. int channels = src_quantization->scale()->size(); TfLiteAffineQuantization* quantization = - reinterpret_cast( - allocator->AllocateFromTail(sizeof(TfLiteAffineQuantization), - alignof(TfLiteAffineQuantization))); + allocate_temp + ? reinterpret_cast( + allocator->AllocateTemp(sizeof(TfLiteAffineQuantization), + alignof(TfLiteAffineQuantization))) + : reinterpret_cast( + allocator->AllocateFromTail( + sizeof(TfLiteAffineQuantization), + alignof(TfLiteAffineQuantization))); if (quantization == nullptr) { TF_LITE_REPORT_ERROR(error_reporter, "Unable to allocate TfLiteAffineQuantization.\n"); return kTfLiteError; } + + // TODO(b/153688719): Reduce tail allocation by using a global zero-point + // buffer. This value can not be reused from the flatbuffer since the + // zero_point is stored as a int64_t. quantization->zero_point = - reinterpret_cast(allocator->AllocateFromTail( - TfLiteIntArrayGetSizeInBytes(channels), alignof(TfLiteIntArray))); + allocate_temp + ? reinterpret_cast(allocator->AllocateTemp( + TfLiteIntArrayGetSizeInBytes(channels), + alignof(TfLiteIntArray))) + : reinterpret_cast(allocator->AllocateFromTail( + TfLiteIntArrayGetSizeInBytes(channels), + alignof(TfLiteIntArray))); if (quantization->zero_point == nullptr) { TF_LITE_REPORT_ERROR(error_reporter, "Unable to allocate quantization->zero_point.\n"); return kTfLiteError; } - quantization->scale = reinterpret_cast( - allocator->AllocateFromTail(TfLiteFloatArrayGetSizeInBytes(channels), - alignof(TfLiteFloatArray))); - if (quantization->scale == nullptr) { - TF_LITE_REPORT_ERROR(error_reporter, - "Unable to allocate quantization->scale.\n"); - return kTfLiteError; - } + TF_LITE_ENSURE_STATUS(FlatBufferVectorToTfLiteTypeArray( + allocator, error_reporter, src_quantization->scale(), + &quantization->scale)); quantization->zero_point->size = channels; - quantization->scale->size = channels; int* zero_point_data = quantization->zero_point->data; - float* scale_data = quantization->scale->data; for (int i = 0; i < channels; i++) { zero_point_data[i] = src_quantization->zero_point()->Get(i); - scale_data[i] = src_quantization->scale()->Get(i); } // TODO(rocky): Need to add a micro_allocator test case that fails when // this is not copied: @@ -373,93 +567,199 @@ TfLiteStatus InitializeRuntimeTensor( result->quantization = {kTfLiteAffineQuantization, quantization}; } - if (flatbuffer_tensor.name() != nullptr) { - result->name = flatbuffer_tensor.name()->c_str(); + return kTfLiteOk; +} + +TfLiteStatus InitializeTfLiteEvalTensorFromFlatbuffer( + SimpleMemoryAllocator* allocator, const tflite::Tensor& flatbuffer_tensor, + const flatbuffers::Vector>* buffers, + ErrorReporter* error_reporter, TfLiteEvalTensor* result) { + *result = {}; + // Make sure the serialized type is one we know how to deal with, and convert + // it from a flatbuffer enum into a constant used by the kernel C API. + TF_LITE_ENSURE_STATUS(ConvertTensorType(flatbuffer_tensor.type(), + &result->type, error_reporter)); + + result->data.data = GetFlatbufferTensorBuffer(flatbuffer_tensor, buffers); + + if (flatbuffer_tensor.shape() == nullptr) { + // flatbuffer_tensor.shape() can return a nullptr in the case of a scalar + // tensor. + result->dims = const_cast(&kZeroLengthIntArray); + } else { + TF_LITE_ENSURE_STATUS(FlatBufferVectorToTfLiteTypeArray( + allocator, error_reporter, flatbuffer_tensor.shape(), &(result->dims))); } return kTfLiteOk; } + } // namespace internal -TfLiteStatus MicroAllocator::Init() { - auto* subgraphs = model_->subgraphs(); - if (subgraphs->size() != 1) { +MicroAllocator::MicroAllocator(SimpleMemoryAllocator* memory_allocator, + ErrorReporter* error_reporter) + : memory_allocator_(memory_allocator), + error_reporter_(error_reporter), + model_is_allocating_(false) {} + +MicroAllocator::~MicroAllocator() {} + +MicroAllocator* MicroAllocator::Create(uint8_t* tensor_arena, size_t arena_size, + ErrorReporter* error_reporter) { + uint8_t* aligned_arena = AlignPointerUp(tensor_arena, kBufferAlignment); + size_t aligned_arena_size = tensor_arena + arena_size - aligned_arena; + return Create(SimpleMemoryAllocator::Create(error_reporter, aligned_arena, + aligned_arena_size), + error_reporter); +} + +MicroAllocator* MicroAllocator::Create(SimpleMemoryAllocator* memory_allocator, + ErrorReporter* error_reporter) { + TFLITE_DCHECK(memory_allocator != nullptr); + TFLITE_DCHECK(error_reporter != nullptr); + + uint8_t* allocator_buffer = memory_allocator->AllocateFromTail( + sizeof(MicroAllocator), alignof(MicroAllocator)); + MicroAllocator* allocator = + new (allocator_buffer) MicroAllocator(memory_allocator, error_reporter); + return allocator; +} + +TfLiteStatus MicroAllocator::StartModelAllocation( + const Model* model, const MicroOpResolver& op_resolver, + NodeAndRegistration** node_and_registrations, + TfLiteEvalTensor** eval_tensors) { + TFLITE_DCHECK(model != nullptr); + + if (model_is_allocating_) { TF_LITE_REPORT_ERROR(error_reporter_, - "Only 1 subgraph is currently supported.\n"); - return kTfLiteError; - } - subgraph_ = (*subgraphs)[0]; - tensors_ = subgraph_->tensors(); - operators_ = subgraph_->operators(); - - context_->tensors_size = tensors_->size(); - context_->tensors = - reinterpret_cast(memory_allocator_->AllocateFromTail( - sizeof(TfLiteTensor) * context_->tensors_size, - alignof(TfLiteTensor))); - if (context_->tensors == nullptr) { - TF_LITE_REPORT_ERROR( - error_reporter_, - "Failed to allocate memory for context->tensors, %d bytes required", - sizeof(TfLiteTensor) * context_->tensors_size); + "MicroAllocator: Model allocation started before " + "finishing previously allocated model"); return kTfLiteError; } - // Initialize runtime tensors in context_ using the flatbuffer. - for (size_t i = 0; i < tensors_->size(); ++i) { - TfLiteStatus status = internal::InitializeRuntimeTensor( - memory_allocator_, *tensors_->Get(i), model_->buffers(), - error_reporter_, &context_->tensors[i]); - if (status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter_, "Failed to initialize tensor %d", - i); - return kTfLiteError; + model_is_allocating_ = true; + + TF_LITE_ENSURE_STATUS(InitScratchBufferData()); + TF_LITE_ENSURE_STATUS(AllocateTfLiteEvalTensors(model, eval_tensors)); + TF_LITE_ENSURE_STATUS( + AllocateNodeAndRegistrations(model, node_and_registrations)); + TF_LITE_ENSURE_STATUS(PrepareNodeAndRegistrationDataFromFlatbuffer( + model, op_resolver, *node_and_registrations)); + + return kTfLiteOk; +} + +TfLiteStatus MicroAllocator::FinishModelAllocation( + const Model* model, TfLiteEvalTensor* eval_tensors, + ScratchBufferHandle** scratch_buffer_handles) { + if (!model_is_allocating_) { + TF_LITE_REPORT_ERROR(error_reporter_, + "MicroAllocator: Model allocation finished before " + "starting allocating model"); + return kTfLiteError; + } + + const SubGraph* subgraph = GetSubGraphFromModel(model); + TFLITE_DCHECK(subgraph != nullptr); + + TF_LITE_ENSURE_STATUS(AllocateScratchBufferHandles( + scratch_buffer_handles, scratch_buffer_request_count_)); + TF_LITE_ENSURE_STATUS(CommitStaticMemoryPlan(model, subgraph, eval_tensors, + *scratch_buffer_handles)); + TF_LITE_ENSURE_STATUS(AllocateVariables(subgraph, eval_tensors)); + + model_is_allocating_ = false; + return kTfLiteOk; +} + +void* MicroAllocator::AllocatePersistentBuffer(size_t bytes) { + return memory_allocator_->AllocateFromTail(bytes, kBufferAlignment); +} + +TfLiteStatus MicroAllocator::RequestScratchBufferInArena(size_t bytes, + int* buffer_idx) { + // All scratch buffer requests are stored in the head section of the arena + // when a model is in the prepare phase. First align a scratch buffer request + // pointer to the start of the head: + internal::ScratchBufferRequest* requests = GetScratchBufferRequests(); + + // Count the number of requested scratch buffers for the current node: + size_t current_node_request_count = 0; + for (size_t i = 0; i < scratch_buffer_request_count_; ++i) { + if (requests[i].node_idx == kUnassignedScratchBufferRequestIndex) { + ++current_node_request_count; } } - return kTfLiteOk; -} - -MicroAllocator::MicroAllocator(TfLiteContext* context, const Model* model, - uint8_t* tensor_arena, size_t arena_size, - ErrorReporter* error_reporter) - : model_(model), error_reporter_(error_reporter), context_(context) { - uint8_t* aligned_arena = AlignPointerUp(tensor_arena, kBufferAlignment); - if (aligned_arena != tensor_arena) { + // First, ensure that the per-kernel request has not exceeded the limit: + if (current_node_request_count >= kMaxScratchBuffersPerOp) { TF_LITE_REPORT_ERROR( error_reporter_, - "%d bytes lost due to alignment. To avoid this loss, please make sure " - "the tensor_arena is 16 bytes aligned.", - aligned_arena - tensor_arena); - } - size_t aligned_arena_size = tensor_arena + arena_size - aligned_arena; - // Creates a root memory allocator managing the arena. The allocator itself - // also locates in the arena buffer. This allocator doesn't need to be - // destructed as it's the root allocator. - memory_allocator_ = CreateInPlaceSimpleMemoryAllocator( - error_reporter, aligned_arena, aligned_arena_size); - TfLiteStatus status = Init(); - // TODO(b/147871299): Consider improving this code. A better way of handling - // failures in the constructor is to have a static function that returns a - // pointer to the class. If allocation failed, a nullptr will be returned. - if (status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter_, - "MicroAllocator: Failed to initialize."); - active_ = false; - } else { - active_ = true; - } -} - -TfLiteStatus MicroAllocator::AllocateNodeAndRegistrations( - const OpResolver& op_resolver, - NodeAndRegistration** node_and_registrations) { - if (!active_) { + "Scratch buffer request exeeds limit per operator (%d)", + kMaxScratchBuffersPerOp); return kTfLiteError; } - auto* output = reinterpret_cast( + // Initialize and assign values for the request at the current index: + internal::ScratchBufferRequest* current_request = + &requests[scratch_buffer_request_count_]; + *current_request = {}; + // Assign -1 as a sentinel value that will be updated when the node finishes + // allocating: + current_request->bytes = bytes; + current_request->node_idx = kUnassignedScratchBufferRequestIndex; + + // Assign the current request index to the out-param: + *buffer_idx = scratch_buffer_request_count_; + + // Bump the request count to prepare for the next request: + ++scratch_buffer_request_count_; + return kTfLiteOk; +} + +TfLiteStatus MicroAllocator::FinishPrepareNodeAllocations(int node_id) { + // When a node has finished preparing, all temp allocations performed by the + // kernel should be cleaned up: + ResetTempAllocations(); + + // Find and update any new scratch buffer requests for the current node: + internal::ScratchBufferRequest* requests = GetScratchBufferRequests(); + + for (size_t i = 0; i < scratch_buffer_request_count_; ++i) { + // A request with a node_idx of -1 is a sentinel value used to indicate this + // was a new request for the current node. The allocator finally knows the + // node index at this point. Assign the value and update the list of new + // requests so the head section can be adjusted to allow for the next kernel + // to allocate at most kMaxScratchBuffersPerOp requests: + if (requests[i].node_idx == kUnassignedScratchBufferRequestIndex) { + requests[i].node_idx = node_id; + } + } + + // Ensure that the head is re-adjusted to allow for another at-most + // kMaxScratchBuffersPerOp scratch buffer requests in the next operator: + TF_LITE_ENSURE_STATUS(memory_allocator_->SetHeadBufferSize( + sizeof(internal::ScratchBufferRequest) * + (scratch_buffer_request_count_ + kMaxScratchBuffersPerOp), + alignof(internal::ScratchBufferRequest))); + + return kTfLiteOk; +} + +size_t MicroAllocator::used_bytes() const { + return memory_allocator_->GetUsedBytes(); +} + +TfLiteStatus MicroAllocator::AllocateNodeAndRegistrations( + const Model* model, NodeAndRegistration** node_and_registrations) { + TFLITE_DCHECK(node_and_registrations); + + const SubGraph* subgraph = GetSubGraphFromModel(model); + TFLITE_DCHECK(subgraph != nullptr); + + NodeAndRegistration* output = reinterpret_cast( memory_allocator_->AllocateFromTail( - sizeof(NodeAndRegistration) * operators_->size(), + sizeof(NodeAndRegistration) * subgraph->operators()->size(), alignof(NodeAndRegistration))); if (output == nullptr) { TF_LITE_REPORT_ERROR( @@ -467,27 +767,41 @@ TfLiteStatus MicroAllocator::AllocateNodeAndRegistrations( "Failed to allocate memory for node_and_registrations."); return kTfLiteError; } + *node_and_registrations = output; + return kTfLiteOk; +} + +TfLiteStatus MicroAllocator::PrepareNodeAndRegistrationDataFromFlatbuffer( + const Model* model, const MicroOpResolver& op_resolver, + NodeAndRegistration* node_and_registrations) { + TFLITE_DCHECK(model != nullptr); + TFLITE_DCHECK(node_and_registrations != nullptr); + + const SubGraph* subgraph = GetSubGraphFromModel(model); + TFLITE_DCHECK(subgraph != nullptr); + TfLiteStatus status = kTfLiteOk; - auto* opcodes = model_->operator_codes(); + auto* opcodes = model->operator_codes(); MicroBuiltinDataAllocator builtin_data_allocator(memory_allocator_); - for (size_t i = 0; i < operators_->size(); ++i) { - const auto* op = operators_->Get(i); - size_t index = op->opcode_index(); + for (size_t i = 0; i < subgraph->operators()->size(); ++i) { + const auto* op = subgraph->operators()->Get(i); + const size_t index = op->opcode_index(); if (index >= opcodes->size()) { TF_LITE_REPORT_ERROR(error_reporter_, "Missing registration for opcode_index %d\n", index); return kTfLiteError; } auto* opcode = (*opcodes)[index]; - status = GetRegistrationFromOpCode(opcode, op_resolver, error_reporter_, - &(output[i].registration)); + status = + GetRegistrationFromOpCode(opcode, op_resolver, error_reporter_, + &(node_and_registrations[i].registration)); if (status != kTfLiteOk) { TF_LITE_REPORT_ERROR(error_reporter_, "Failed to get registration from op code %s\n ", - EnumNameBuiltinOperator(opcode->builtin_code())); + EnumNameBuiltinOperator(GetBuiltinCode(opcode))); return status; } - const auto* registration = output[i].registration; + const auto* registration = node_and_registrations[i].registration; if (registration == nullptr) { TF_LITE_REPORT_ERROR(error_reporter_, "Skipping op for opcode_index %d\n", index); @@ -496,34 +810,48 @@ TfLiteStatus MicroAllocator::AllocateNodeAndRegistrations( BuiltinOperator op_type = static_cast(registration->builtin_code); - if (op_type != BuiltinOperator_CUSTOM && op->custom_options()) { - TF_LITE_REPORT_ERROR( - error_reporter_, - "Unsupported behavior: found builtin operator %s with custom " - "options.\n", - EnumNameBuiltinOperator(op_type)); - return kTfLiteError; - } - const char* custom_data = nullptr; size_t custom_data_size = 0; unsigned char* builtin_data = nullptr; - if (op->custom_options()) { - custom_data = reinterpret_cast(op->custom_options()->data()); - custom_data_size = op->custom_options()->size(); + + if (op_type == BuiltinOperator_CUSTOM) { + // Custom Ops may or may not have a non-null custom_options field. + if (op->custom_options() != nullptr) { + custom_data = + reinterpret_cast(op->custom_options()->data()); + custom_data_size = op->custom_options()->size(); + } } else { - TF_LITE_ENSURE_STATUS(ParseOpData(op, op_type, error_reporter_, - &builtin_data_allocator, - (void**)(&builtin_data))); + if (op->custom_options() != nullptr) { + TF_LITE_REPORT_ERROR( + error_reporter_, + "Unsupported behavior: found builtin operator %s with custom " + "options.\n", + EnumNameBuiltinOperator(op_type)); + return kTfLiteError; + } + + MicroOpResolver::BuiltinParseFunction parser = + op_resolver.GetOpDataParser(op_type); + if (parser == nullptr) { + TF_LITE_REPORT_ERROR(error_reporter_, "Did not find a parser for %s", + EnumNameBuiltinOperator(op_type)); + + return kTfLiteError; + } + TF_LITE_ENSURE_STATUS(parser(op, error_reporter_, &builtin_data_allocator, + (void**)(&builtin_data))); } - // Disregard const qualifier to workaround with existing API. - TfLiteIntArray* inputs_array = const_cast( - reinterpret_cast(op->inputs())); - TfLiteIntArray* outputs_array = const_cast( - reinterpret_cast(op->outputs())); + TfLiteIntArray* inputs_array; + TF_LITE_ENSURE_STATUS(internal::FlatBufferVectorToTfLiteTypeArray( + memory_allocator_, error_reporter_, op->inputs(), &inputs_array)); - TfLiteNode* node = &(output[i].node); + TfLiteIntArray* outputs_array; + TF_LITE_ENSURE_STATUS(internal::FlatBufferVectorToTfLiteTypeArray( + memory_allocator_, error_reporter_, op->outputs(), &outputs_array)); + + TfLiteNode* node = &(node_and_registrations[i].node); *node = {}; node->inputs = inputs_array; node->outputs = outputs_array; @@ -531,139 +859,300 @@ TfLiteStatus MicroAllocator::AllocateNodeAndRegistrations( node->custom_initial_data = custom_data; node->custom_initial_data_size = custom_data_size; } - *node_and_registrations = output; + return kTfLiteOk; } -TfLiteStatus MicroAllocator::FinishTensorAllocation() { - if (!active_) { +TfLiteTensor* MicroAllocator::AllocatePersistentTfLiteTensor( + const Model* model, TfLiteEvalTensor* eval_tensors, int tensor_index) { + const SubGraph* subgraph = GetSubGraphFromModel(model); + TFLITE_DCHECK(subgraph != nullptr); + + // This value is allocated from persistent arena space. It is guaranteed to be + // around for the lifetime of the application. + TfLiteTensor* tensor = + AllocatePersistentTfLiteTensorInternal(model, eval_tensors, tensor_index); + + // Populate any fields from the flatbuffer, since this TfLiteTensor struct is + // allocated in the persistent section of the arena, ensure that additional + // allocations also take place in that section of the arena. + if (PopulateTfLiteTensorFromFlatbuffer(model, subgraph, tensor, tensor_index, + /*allocate_temp=*/false) != + kTfLiteOk) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Failed to populate a persistent TfLiteTensor struct " + "from flatbuffer data!"); + return nullptr; + } + + if (eval_tensors != nullptr) { + // Tensor buffers that are allocated at runtime (e.g. non-weight buffers) + // and not located in the flatbuffer are stored on the pre-allocated list of + // TfLiteEvalTensors structs. These structs are the source of truth, simply + // point the corresponding buffer to the new TfLiteTensor data value. + tensor->data.data = eval_tensors[tensor_index].data.data; + } + return tensor; +} + +TfLiteTensor* MicroAllocator::AllocateTempTfLiteTensor( + const Model* model, TfLiteEvalTensor* eval_tensors, int tensor_index) { + const SubGraph* subgraph = GetSubGraphFromModel(model); + TFLITE_DCHECK(subgraph != nullptr); + + // This value is allocated from temporary arena space. It is guaranteed to be + // around for at least the scope of the calling function. Since this struct + // allocation takes place in temp space, no need to own or cleanup. + TfLiteTensor* tensor = + reinterpret_cast(memory_allocator_->AllocateTemp( + sizeof(TfLiteTensor), alignof(TfLiteTensor))); + + // Populate any fields from the flatbuffer, since this TfLiteTensor struct is + // allocated in the temp section of the arena, ensure that additional + // allocations also take place in that section of the arena. + if (PopulateTfLiteTensorFromFlatbuffer(model, subgraph, tensor, tensor_index, + /*allocate_temp=*/true) != kTfLiteOk) { + TF_LITE_REPORT_ERROR( + error_reporter_, + "Failed to populate a temp TfLiteTensor struct from flatbuffer data!"); + return nullptr; + } + + if (eval_tensors != nullptr) { + // Tensor buffers that are allocated at runtime (e.g. non-weight buffers) + // and not located in the flatbuffer are stored on the pre-allocated list of + // TfLiteEvalTensors structs. These structs are the source of truth, simply + // point the corresponding buffer to the new TfLiteTensor data value. + tensor->data.data = eval_tensors[tensor_index].data.data; + } + return tensor; +} + +void MicroAllocator::ResetTempAllocations() { + memory_allocator_->ResetTempAllocations(); +} + +TfLiteStatus MicroAllocator::AllocateTfLiteEvalTensors( + const Model* model, TfLiteEvalTensor** eval_tensors) { + TFLITE_DCHECK(eval_tensors != nullptr); + + const SubGraph* subgraph = GetSubGraphFromModel(model); + TFLITE_DCHECK(subgraph != nullptr); + + size_t alloc_count = subgraph->tensors()->size(); + TfLiteEvalTensor* tensors = + reinterpret_cast(memory_allocator_->AllocateFromTail( + sizeof(TfLiteEvalTensor) * alloc_count, alignof(TfLiteEvalTensor))); + if (tensors == nullptr) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Failed to allocate memory for context->eval_tensors, " + "%d bytes required", + sizeof(TfLiteEvalTensor) * alloc_count); return kTfLiteError; } + for (size_t i = 0; i < alloc_count; ++i) { + TfLiteStatus status = internal::InitializeTfLiteEvalTensorFromFlatbuffer( + memory_allocator_, *subgraph->tensors()->Get(i), model->buffers(), + error_reporter_, &tensors[i]); + if (status != kTfLiteOk) { + TF_LITE_REPORT_ERROR(error_reporter_, "Failed to initialize tensor %d", + i); + return kTfLiteError; + } + } + *eval_tensors = tensors; + return kTfLiteOk; +} + +TfLiteStatus MicroAllocator::AllocateVariables(const SubGraph* subgraph, + TfLiteEvalTensor* eval_tensors) { + for (size_t i = 0; i < subgraph->tensors()->size(); ++i) { + auto* tensor = subgraph->tensors()->Get(i); + if (tensor->is_variable()) { + size_t buffer_size; + TF_LITE_ENSURE_STATUS( + TfLiteEvalTensorByteLength(&eval_tensors[i], &buffer_size)); + + eval_tensors[i].data.data = + memory_allocator_->AllocateFromTail(buffer_size, kBufferAlignment); + + if (eval_tensors[i].data.data == nullptr) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Failed to allocate variable tensor of size %d", + buffer_size); + return kTfLiteError; + } + } + } + return kTfLiteOk; +} + +TfLiteTensor* MicroAllocator::AllocatePersistentTfLiteTensorInternal( + const Model* model, TfLiteEvalTensor* eval_tensors, int tensor_index) { + return reinterpret_cast(memory_allocator_->AllocateFromTail( + sizeof(TfLiteTensor), alignof(TfLiteTensor))); +} + +TfLiteStatus MicroAllocator::PopulateTfLiteTensorFromFlatbuffer( + const Model* model, const SubGraph* subgraph, TfLiteTensor* tensor, + int tensor_index, bool allocate_temp) { + // TODO(b/162311891): This method serves as a stub to ensure quantized + // allocations in the tail can be recorded. Once the interpreter has APIs for + // accessing buffers on TfLiteEvalTensor this method can be dropped. + return internal::InitializeTfLiteTensorFromFlatbuffer( + memory_allocator_, allocate_temp, *subgraph->tensors()->Get(tensor_index), + model->buffers(), error_reporter_, tensor); +} + +ErrorReporter* MicroAllocator::error_reporter() const { + return error_reporter_; +} + +const SubGraph* MicroAllocator::GetSubGraphFromModel(const Model* model) { + auto* subgraphs = model->subgraphs(); + if (subgraphs->size() != 1) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Only 1 subgraph is currently supported.\n"); + return nullptr; + } + return (*subgraphs)[0]; +} + +TfLiteStatus MicroAllocator::CommitStaticMemoryPlan( + const Model* model, const SubGraph* subgraph, + TfLiteEvalTensor* eval_tensors, + ScratchBufferHandle* scratch_buffer_handles) { + size_t head_usage = 0; // Create static memory plan // 1. Calculate AllocationInfo to know the lifetime of each tensor/buffer. // 2. Add them into the planner (such as the GreedyMemoryPlanner). // 3. Static memory planning using the planner. // 4. Set tensor/buffer pointers based on the offsets from the previous step. + // // Note that AllocationInfo is only needed for creating the plan. It will be - // thrown away when the child allocator (tmp_allocator) goes out of scope. - { - SimpleMemoryAllocator tmp_allocator(error_reporter_, - memory_allocator_->GetHead(), - memory_allocator_->GetTail()); + // allocated from the temp section and cleaned up at the bottom of this + // function. - AllocationInfoBuilder builder(error_reporter_, &tmp_allocator); - TF_LITE_ENSURE_STATUS( - builder.Init(tensors_->size(), scratch_buffer_count_)); - TF_LITE_ENSURE_STATUS(builder.AddTensors(subgraph_, context_->tensors)); - TF_LITE_ENSURE_STATUS(builder.AddScratchBuffers(scratch_buffer_handles_)); - const AllocationInfo* allocation_info = builder.Finish(); + size_t allocation_info_count = + subgraph->tensors()->size() + scratch_buffer_request_count_; + size_t bytes = sizeof(AllocationInfo) * allocation_info_count; - // Remaining arena size that memory planner can use for calculating offsets. - size_t remaining_arena_size = tmp_allocator.GetAvailableMemory(); - uint8_t* planner_arena = - tmp_allocator.AllocateFromHead(remaining_arena_size, /*alignment=*/1); - TF_LITE_ENSURE(error_reporter_, planner_arena != nullptr); - GreedyMemoryPlanner planner(planner_arena, remaining_arena_size); - TF_LITE_ENSURE_STATUS( - CreatePlan(error_reporter_, &planner, allocation_info, builder.Size())); - - size_t actual_available_arena_size = - memory_allocator_->GetAvailableMemory(); - // Make sure we have enough arena size. - if (planner.GetMaximumMemorySize() > actual_available_arena_size) { - TF_LITE_REPORT_ERROR( - error_reporter_, - "Arena size is too small for activation buffers. Needed %d but only " - "%d was available.", - planner.GetMaximumMemorySize(), actual_available_arena_size); - return kTfLiteError; - } - - // Commit the plan. - TF_LITE_ENSURE_STATUS(CommitPlan(error_reporter_, &planner, - memory_allocator_->GetHead(), - allocation_info, builder.Size())); - // Allocate the planned area, so the allocator knows it's used. - uint8_t* allocated_tensor_memory = - memory_allocator_->AllocateFromHead(planner.GetMaximumMemorySize(), - /*alignment=*/1); - TF_LITE_ENSURE(error_reporter_, allocated_tensor_memory != nullptr); - } - - // Data in variables need to be kept for the next invocation so allocating - // them from the tail (persistent area). - if (AllocateVariables(tensors_, context_->tensors, memory_allocator_) != - kTfLiteOk) { + // Allocate an array of AllocationInfo structs from the temp section. This + // struct will be used by AllocationInfoBuilder to find buffer usage. + AllocationInfo* allocation_info = reinterpret_cast( + memory_allocator_->AllocateTemp(bytes, alignof(AllocationInfo))); + if (allocation_info == nullptr) { TF_LITE_REPORT_ERROR( error_reporter_, - "Failed to allocate variables. Please increase arena size."); + "Failed to allocate memory for allocation_info, %d bytes required", + bytes); return kTfLiteError; } - active_ = false; + // Use the AllocationInfoBuilder class to help determine where buffers are + // used in the subgraph. + AllocationInfoBuilder builder(allocation_info, subgraph->tensors()->size(), + scratch_buffer_request_count_, error_reporter_); + + const int32_t* offline_planner_offsets = nullptr; + TF_LITE_ENSURE_STATUS( + builder.GetOfflinePlannedOffsets(model, &offline_planner_offsets)); + TF_LITE_ENSURE_STATUS( + builder.AddTensors(subgraph, offline_planner_offsets, eval_tensors)); + + internal::ScratchBufferRequest* scratch_buffer_requests = + GetScratchBufferRequests(); + + TF_LITE_ENSURE_STATUS(builder.AddScratchBuffers(scratch_buffer_requests, + scratch_buffer_handles)); + + // Remaining arena size that memory planner can use for calculating offsets. + size_t remaining_arena_size = + memory_allocator_->GetAvailableMemory(kBufferAlignment); + uint8_t* planner_arena = + memory_allocator_->AllocateTemp(remaining_arena_size, kBufferAlignment); + TF_LITE_ENSURE(error_reporter_, planner_arena != nullptr); + GreedyMemoryPlanner planner(planner_arena, remaining_arena_size); + TF_LITE_ENSURE_STATUS(CreatePlan(error_reporter_, &planner, allocation_info, + allocation_info_count)); + + // Reset all temp allocations used above: + memory_allocator_->ResetTempAllocations(); + + size_t actual_available_arena_size = + memory_allocator_->GetAvailableMemory(kBufferAlignment); + + // Make sure we have enough arena size. + if (planner.GetMaximumMemorySize() > actual_available_arena_size) { + TF_LITE_REPORT_ERROR( + error_reporter_, + "Arena size is too small for all buffers. Needed %u but only " + "%u was available.", + planner.GetMaximumMemorySize(), actual_available_arena_size); + return kTfLiteError; + } + // Commit the plan. + TF_LITE_ENSURE_STATUS(CommitPlan(error_reporter_, &planner, + memory_allocator_->GetHeadBuffer(), + allocation_info, allocation_info_count)); + head_usage = planner.GetMaximumMemorySize(); + + // The head is used to store memory plans for one model at a time during the + // model preparation stage, and is re-purposed to store scratch buffer handles + // during model invocation. The head must be as large as the greater of the + // largest model memory plan's size and the total space required for all + // scratch buffer handles. + if (max_head_buffer_usage_ < head_usage) { + max_head_buffer_usage_ = head_usage; + } + + // The head is used for storing scratch buffer allocations before finalizing a + // memory plan in this function. Ensure that the head is set to the largest + // memory plan sent through the allocator: + TF_LITE_ENSURE_STATUS(memory_allocator_->SetHeadBufferSize( + max_head_buffer_usage_, kBufferAlignment)); return kTfLiteOk; } -TfLiteStatus MicroAllocator::AllocatePersistentBuffer(size_t bytes, - void** ptr) { - uint8_t* data = memory_allocator_->AllocateFromTail(bytes, kBufferAlignment); - if (data == nullptr) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Failed to allocate persistent buffer of size %d", - bytes); - return kTfLiteError; +TfLiteStatus MicroAllocator::AllocateScratchBufferHandles( + ScratchBufferHandle** scratch_buffer_handles, size_t handle_count) { + TFLITE_DCHECK(scratch_buffer_handles != nullptr); + + if (scratch_buffer_request_count_ == 0) { + // No scratch buffer requests were requested during model allocation. + return kTfLiteOk; } - (*ptr) = data; + + // Allocate a consecutive block of memory store the scratch buffer handles. + // This alignment ensures quick lookup during inference time for the model: + *scratch_buffer_handles = reinterpret_cast( + memory_allocator_->AllocateFromTail( + sizeof(ScratchBufferHandle) * handle_count, + alignof(ScratchBufferHandle))); + return kTfLiteOk; } -TfLiteStatus MicroAllocator::RequestScratchBufferInArena(int node_id, - size_t bytes, - int* buffer_idx) { - // A sanity check to make sure scratch_buffer_handles_ is contiguous i.e. - // scratch_buffer_handles_ is pointing to the last allocation from memory - // allocator. - if (scratch_buffer_handles_ != nullptr && - reinterpret_cast(scratch_buffer_handles_) != - memory_allocator_->GetTail()) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Internal error: AllocateFromTail can not be called " - "between two RequestScratchBufferInArena calls."); - return kTfLiteError; - } +TfLiteStatus MicroAllocator::InitScratchBufferData() { + // A model is preparing to allocate resources, ensure that scratch buffer + // request counter is cleared: + scratch_buffer_request_count_ = 0; + + // All requests will be stored in the head section. Each kernel is allowed at + // most kMaxScratchBuffersPerOp requests. Adjust the head to reserve at most + // that many requests to begin: + TF_LITE_ENSURE_STATUS(memory_allocator_->SetHeadBufferSize( + sizeof(internal::ScratchBufferRequest) * kMaxScratchBuffersPerOp, + alignof(internal::ScratchBufferRequest))); - internal::ScratchBufferHandle* handle = - reinterpret_cast( - memory_allocator_->AllocateFromTail( - sizeof(internal::ScratchBufferHandle), - alignof(internal::ScratchBufferHandle))); - if (handle == nullptr) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Failed to register scratch buffer handle for node %s", - node_id); - return kTfLiteError; - } - *handle = {}; - handle->bytes = bytes; - handle->node_idx = node_id; - *buffer_idx = scratch_buffer_count_; - scratch_buffer_count_ += 1; - // scratch_buffer_handles_ is in reverse order. The following code ensures - // that scratch_buffers[0] is pointing to the newly allocated handle. - scratch_buffer_handles_ = handle; return kTfLiteOk; } -void* MicroAllocator::GetScratchBuffer(int buffer_idx) const { - if (static_cast(buffer_idx) >= scratch_buffer_count_) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Buffer %d not found. %d buffers available.", - buffer_idx, scratch_buffer_count_); - return nullptr; - } - // scratch_buffer_handles_ is in reverse order. - return scratch_buffer_handles_[scratch_buffer_count_ - buffer_idx - 1].data; +internal::ScratchBufferRequest* MicroAllocator::GetScratchBufferRequests() { + return reinterpret_cast( + AlignPointerUp(memory_allocator_->GetHeadBuffer(), + alignof(internal::ScratchBufferRequest))); } } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_allocator.h b/code/lib/tfmicro/tensorflow/lite/micro/micro_allocator.h index a846b0c6..39a12eaf 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_allocator.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_allocator.h @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,33 +15,42 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_MICRO_ALLOCATOR_H_ #define TENSORFLOW_LITE_MICRO_MICRO_ALLOCATOR_H_ +#include +#include + +#include "flatbuffers/flatbuffers.h" // from @flatbuffers #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" -#include "tensorflow/lite/core/api/flatbuffer_conversions.h" +#include "tensorflow/lite/micro/compatibility.h" +#include "tensorflow/lite/micro/micro_op_resolver.h" #include "tensorflow/lite/micro/simple_memory_allocator.h" #include "tensorflow/lite/schema/schema_generated.h" namespace tflite { -// Namespace used for unittests. namespace internal { -// Sets up all of the data structure members for a runtime tensor -// based on the contents of a serialized tensor. -TfLiteStatus InitializeRuntimeTensor( - SimpleMemoryAllocator* allocator, const tflite::Tensor& flatbuffer_tensor, +// Sets up all of the data structure members for a TfLiteTensor based on the +// contents of a serialized tensor in the flatbuffer. +// TODO(b/162311891): Drop this method when the interpreter has an API for +// returning buffers on TfLiteEvalTensor. +TfLiteStatus InitializeTfLiteTensorFromFlatbuffer( + SimpleMemoryAllocator* allocator, bool allocate_temp, + const tflite::Tensor& flatbuffer_tensor, const flatbuffers::Vector>* buffers, ErrorReporter* error_reporter, TfLiteTensor* result); -// A handle tracking scratch buffer allocation. This handle is created by -// `RequestScratchBufferInArena`. `data` field is populated in -// `FinishTensorAllocation` after static memory planning. -// TODO(b/150257460) As a future optimization, this struct could be replaced by -// a union, since once `data` is populated, `bytes` and `node_idx` is not -// needed. +// Holds placeholder information for a scratch buffer request from a kernel. +// This struct is only used during the model prepare stage. Each request from a +// kernel is stored in the head section. During the prepare stage, the head +// section will at least hold kMaxScratchBuffersPerOp number of requests plus +// any requests from previous kernel requests. +// +// When the memory plan is finalized, these structs are no longer used in favor +// of a sequential, array of ScratchBufferHandle allocations in the tail +// section. These allocations are indexed by the request API defined in the +// TfLiteContext struct. typedef struct { - // Pointer to the scratch buffer. - uint8_t* data; // Number of bytes required by the buffer. The actual allocated size might be // greater than `bytes` due to buffer alignment. size_t bytes; @@ -49,7 +58,8 @@ typedef struct { // determine the lifetime of the buffer. In AllocationInfo, this buffer will // have `before` = node_idx and `after` = node_idx. int node_idx; -} ScratchBufferHandle; +} ScratchBufferRequest; + } // namespace internal typedef struct { @@ -57,9 +67,27 @@ typedef struct { const TfLiteRegistration* registration; } NodeAndRegistration; +// Holds a pointer to a buffer for a scratch buffer requested by a kernel during +// the model prepare stage. This struct is allocated in-place and allows for +// quick pointer-indexed lookup for speed during model inference. +typedef struct { + // Pointer to location of the scratch buffer: + uint8_t* data; +} ScratchBufferHandle; + // Allocator responsible for allocating memory for all intermediate tensors // necessary to invoke a model. - +// +// The lifetime of the model, tensor arena and error reporter must be at +// least as long as that of the allocator object, since the allocator needs +// them to be accessible during its entire lifetime. +// +// The MicroAllocator simply plans out additional allocations that are required +// to standup a model for inference in TF Micro. This class currently relies on +// an additional allocator - SimpleMemoryAllocator - for all allocations from an +// arena. These allocations are divided into head (non-persistent) and tail +// (persistent) regions: +// // Memory layout to help understand how it works // This information could change in the future version. // ************** .memory_allocator->GetBuffer() @@ -72,76 +100,179 @@ typedef struct { // ************** .memory_allocator->GetBuffer() + ->GetMaxBufferSize() class MicroAllocator { public: - // The lifetime of the model, tensor allocator and error reporter must be at - // least as long as that of the allocator object, since the allocator needs - // them to be accessible during its entire lifetime. - + // Creates a MicroAllocator instance from a given tensor arena. This arena + // will be managed by the created instance. // Note: Please use __declspec(align(16)) to make sure tensor_arena is 16 // bytes aligned, otherwise some head room will be wasted. - MicroAllocator(TfLiteContext* context, const Model* model, - uint8_t* tensor_arena, size_t arena_size, - ErrorReporter* error_reporter); + // TODO(b/157615197): Cleanup constructor + factory usage. + static MicroAllocator* Create(uint8_t* tensor_arena, size_t arena_size, + ErrorReporter* error_reporter); - // Runs through the model and allocates all necessary input, output and - // intermediate tensors. - // WARNING: doing any allocation after calling this method has the risk of - // corrupting tensor data so this method should be the last non-const method - // called in this class. - TfLiteStatus FinishTensorAllocation(); + // Creates a MicroAllocator instance using the provided SimpleMemoryAllocator + // intance. This allocator instance will use the SimpleMemoryAllocator + // instance to manage allocations internally. + static MicroAllocator* Create(SimpleMemoryAllocator* memory_allocator, + ErrorReporter* error_reporter); - // Returns the arena usage in bytes, only available after - // `FinishTensorAllocation`. Otherwise, it will return 0. - size_t used_bytes() const { - if (active_) { - return 0; - } - return memory_allocator_->GetUsedBytes(); - } + // Begin allocating internal resources required for model inference. + // This method will run through the flatbuffer data supplied in the model to + // properly allocate tensor, node, and op registration data. This method is + // expected to be followed with a call to FinishModelAllocation() before + // resuming allocation with another model. All persistent tensor buffers are + // stored in the out-param eval_tensors. This value is allocated from the + // persistent memory arena and will be used to host runtime tensor buffers. + TfLiteStatus StartModelAllocation( + const Model* model, const MicroOpResolver& op_resolver, + NodeAndRegistration** node_and_registrations, + TfLiteEvalTensor** eval_tensors); - // Run through the model to allocate nodes and registrations. We need to keep - // them for the entire life time of the model to allow persistent tensors. - // This method needs to be called before FinishTensorAllocation method. - TfLiteStatus AllocateNodeAndRegistrations( - const OpResolver& op_resolver, - NodeAndRegistration** node_and_registrations); + // Finish allocating internal resources required for model inference. + // This method will plan non-persistent buffers and commit a memory plan to + // the 'head' section of the memory arena. All variable tensor data will also + // be allocated. This method should be called after assigning model resources + // in StartModelAllocation(). The eval_tensors pointer should be the value + // passed into this class during StartModelAllocation(). Scratch buffer + // handles are stored in the out-param `scratch_buffer_handles`. This value + // will be used in `GetScratchBuffer` call to retrieve scratch buffers. + TfLiteStatus FinishModelAllocation( + const Model* model, TfLiteEvalTensor* eval_tensors, + ScratchBufferHandle** scratch_buffer_handles); + + // Allocates a TfLiteTensor struct and populates the returned value with + // properties from the model flatbuffer. This struct is allocated from + // persistent arena memory is only guaranteed for the lifetime of the + // application. The eval_tensors pointer should be the value passed into this + // class during StartModelAllocation() and contains the source-of-truth for + // buffers. + virtual TfLiteTensor* AllocatePersistentTfLiteTensor( + const Model* model, TfLiteEvalTensor* eval_tensors, int tensor_index); + + // Allocates a TfLiteTensor struct and populates the returned value with + // properties from the model flatbuffer. This struct is allocated from + // temporary arena memory is only guaranteed until a call is made to + // ResetTempAllocations(). The eval_tensors pointer should be the value passed + // into this class during StartModelAllocation() and contains the + // source-of-truth for buffers. + virtual TfLiteTensor* AllocateTempTfLiteTensor(const Model* model, + TfLiteEvalTensor* eval_tensors, + int tensor_index); + + // Resets all temporary allocations. This method should be called after a + // chain of temp allocations (e.g. chain of TfLiteTensor objects via + // AllocateTfLiteTensor()). + virtual void ResetTempAllocations(); // Allocates persistent buffer which has the same life time as the allocator. // The memory is immediately available and is allocated from the tail of the // arena. - TfLiteStatus AllocatePersistentBuffer(size_t bytes, void** ptr); + virtual void* AllocatePersistentBuffer(size_t bytes); // Register a scratch buffer of size `bytes` for Node with `node_id`. - // This method only allocates a BufferHandle holding information for memory - // planning. The buffer ptr is ready after `FinishTensorAllocation` and can - // be retrieved by `GetScratchBuffer` method using the returned buffer_idx. - // Note that there should be no tail allocation between two consecutive - // `RequestScratchBufferInArena` calls. - TfLiteStatus RequestScratchBufferInArena(int node_id, size_t bytes, - int* buffer_idx); - // Returns the pointer to the planned scratch buffer. - void* GetScratchBuffer(int buffer_idx) const; + // This method only requests a buffer with a given size to be used after a + // model has finished allocation via FinishModelAllocation(). All requested + // buffers will be accessible by the out-param in that method. + TfLiteStatus RequestScratchBufferInArena(size_t bytes, int* buffer_idx); + + // Finish allocating a specific NodeAndRegistration prepare block (kernel + // entry for a model) with a given node ID. This call ensures that any scratch + // buffer requests and temporary allocations are handled and ready for the + // next node prepare block. + TfLiteStatus FinishPrepareNodeAllocations(int node_id); + + // Returns the arena usage in bytes, only available after + // `FinishModelAllocation`. Otherwise, it will return 0. + size_t used_bytes() const; + + protected: + MicroAllocator(SimpleMemoryAllocator* memory_allocator, + ErrorReporter* error_reporter); + virtual ~MicroAllocator(); + + // Allocates an array in the arena to hold pointers to the node and + // registration pointers required to represent the inference graph of the + // model. + virtual TfLiteStatus AllocateNodeAndRegistrations( + const Model* model, NodeAndRegistration** node_and_registrations); + + // Populates node and registration pointers representing the inference graph + // of the model from values inside the flatbuffer (loaded from the TfLiteModel + // instance). Persistent data (e.g. operator data) is allocated from the + // arena. + virtual TfLiteStatus PrepareNodeAndRegistrationDataFromFlatbuffer( + const Model* model, const MicroOpResolver& op_resolver, + NodeAndRegistration* node_and_registrations); + + // Allocates the list of persistent TfLiteEvalTensors that are used for the + // "eval" phase of model inference. These structs will be the source of truth + // for all tensor buffers. Allocation results are stored in the out-param + // eval_tensors. + virtual TfLiteStatus AllocateTfLiteEvalTensors( + const Model* model, TfLiteEvalTensor** eval_tensors); + + // Allocates persistent tensor buffers for variable tensors in the subgraph. + virtual TfLiteStatus AllocateVariables(const SubGraph* subgraph, + TfLiteEvalTensor* eval_tensors); + + // Allocate and return a persistent TfLiteTensor. + // TODO(b/162311891): Drop this method when the interpreter has an API for + // accessing TfLiteEvalTensor structs. + virtual TfLiteTensor* AllocatePersistentTfLiteTensorInternal( + const Model* model, TfLiteEvalTensor* eval_tensors, int tensor_index); + + // Populates a TfLiteTensor struct with data from the model flatbuffer. Any + // quantization data is allocated from either the tail (persistent) or temp + // sections of the arena based on the allocation flag. + virtual TfLiteStatus PopulateTfLiteTensorFromFlatbuffer( + const Model* model, const SubGraph* subgraph, TfLiteTensor* tensor, + int tensor_index, bool allocate_temp); + + ErrorReporter* error_reporter() const; + + // Returns the first subgraph from the model. + const SubGraph* GetSubGraphFromModel(const Model* model); private: - TfLiteStatus Init(); + // Commits a memory plan for all non-persistent buffer allocations in the + // 'head' section of the memory arena. The eval_tensors pointer is the list of + // pre-allocated TfLiteEvalTensor structs that will point to the buffers that + // will be allocated into the head section in this function call. The + // scratch_buffer_handles pointer is the array of pre-allocated + // ScratchBufferHandle structs that will point to allocated buffers also in + // the head section. + virtual TfLiteStatus CommitStaticMemoryPlan( + const Model* model, const SubGraph* subgraph, + TfLiteEvalTensor* eval_tensors, + ScratchBufferHandle* scratch_buffer_handles); - const Model* model_; - // A simple memory allocator that always allocate from the arena tail. + // Allocates an array of ScratchBufferHandle structs in the tail section for a + // given number of handles. + virtual TfLiteStatus AllocateScratchBufferHandles( + ScratchBufferHandle** scratch_buffer_handles, size_t handle_count); + + // Clears all internal scratch buffer request counts and resets the head to + // prepare for kernels to request scratch buffer data when a model is + // preparing. + TfLiteStatus InitScratchBufferData(); + + // Returns the pointer for the array of ScratchBufferRequest allocations in + // the head section. + internal::ScratchBufferRequest* GetScratchBufferRequests(); + + // A simple memory allocator that always allocate from the arena tail or head. SimpleMemoryAllocator* memory_allocator_; + ErrorReporter* error_reporter_; - TfLiteContext* context_; - // Indicating if the allocator is ready for allocation. - bool active_ = false; + bool model_is_allocating_; - // In reverse order for efficiency. - // i.e. scratch_buffer_handles_[0] is the handle for the last buffer, - // corresponding to the last RequestScratchBufferInArena call. - internal::ScratchBufferHandle* scratch_buffer_handles_ = nullptr; - // How many scratch buffers have been allocated. - size_t scratch_buffer_count_ = 0; + // Holds the number of ScratchBufferRequest instances stored in the head + // section when a model is allocating. + size_t scratch_buffer_request_count_ = 0; - const SubGraph* subgraph_; - const flatbuffers::Vector>* operators_; - const flatbuffers::Vector>* tensors_; + // Holds the byte length of the memory plan with the largest head usage. Used + // to ensure that multi-tenant allocations can share the head for buffers. + size_t max_head_buffer_usage_ = 0; + + TF_LITE_REMOVE_VIRTUAL_DELETE }; } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_error_reporter.cc b/code/lib/tfmicro/tensorflow/lite/micro/micro_error_reporter.cc index bea3dc8d..6d8361cd 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_error_reporter.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_error_reporter.cc @@ -15,7 +15,10 @@ limitations under the License. #include "tensorflow/lite/micro/micro_error_reporter.h" +#include + #ifndef TF_LITE_STRIP_ERROR_STRINGS +#include "tensorflow/lite/micro/debug_log.h" #include "tensorflow/lite/micro/micro_string.h" #endif diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_error_reporter.h b/code/lib/tfmicro/tensorflow/lite/micro/micro_error_reporter.h index b18c47f4..e2c073a4 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_error_reporter.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_error_reporter.h @@ -15,9 +15,10 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_MICRO_ERROR_REPORTER_H_ #define TENSORFLOW_LITE_MICRO_MICRO_ERROR_REPORTER_H_ +#include + #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/micro/compatibility.h" -#include "tensorflow/lite/micro/debug_log.h" namespace tflite { diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_interpreter.cc b/code/lib/tfmicro/tensorflow/lite/micro/micro_interpreter.cc index c5d35407..8b003d8b 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_interpreter.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_interpreter.cc @@ -1,4 +1,4 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,16 +14,24 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/micro/micro_interpreter.h" +#include +#include +#include + +#include "flatbuffers/flatbuffers.h" // from @flatbuffers #include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/core/api/flatbuffer_conversions.h" +#include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/core/api/tensor_utils.h" -#include "tensorflow/lite/micro/compatibility.h" +#include "tensorflow/lite/micro/memory_helpers.h" #include "tensorflow/lite/micro/micro_allocator.h" -#include "tensorflow/lite/micro/micro_optional_debug_tools.h" +#include "tensorflow/lite/micro/micro_op_resolver.h" +#include "tensorflow/lite/micro/micro_profiler.h" +#include "tensorflow/lite/schema/schema_generated.h" namespace tflite { namespace { +#ifndef TF_LITE_STRIP_ERROR_STRINGS const char* OpNameFromRegistration(const TfLiteRegistration* registration) { if (registration->builtin_code == BuiltinOperator_CUSTOM) { return registration->custom_name; @@ -31,88 +39,111 @@ const char* OpNameFromRegistration(const TfLiteRegistration* registration) { return EnumNameBuiltinOperator(BuiltinOperator(registration->builtin_code)); } } +#endif // !defined(TF_LITE_STRIP_ERROR_STRINGS) } // namespace namespace internal { -TfLiteStatus ContextHelper::AllocatePersistentBuffer(TfLiteContext* ctx, - size_t bytes, void** ptr) { +ContextHelper::ContextHelper(ErrorReporter* error_reporter, + MicroAllocator* allocator, const Model* model) + : allocator_(allocator), error_reporter_(error_reporter), model_(model) {} + +void* ContextHelper::AllocatePersistentBuffer(TfLiteContext* ctx, + size_t bytes) { return reinterpret_cast(ctx->impl_) - ->allocator_->AllocatePersistentBuffer(bytes, ptr); + ->allocator_->AllocatePersistentBuffer(bytes); } TfLiteStatus ContextHelper::RequestScratchBufferInArena(TfLiteContext* ctx, size_t bytes, int* buffer_idx) { ContextHelper* helper = reinterpret_cast(ctx->impl_); - return helper->allocator_->RequestScratchBufferInArena( - helper->current_node_idx_, bytes, buffer_idx); + return helper->allocator_->RequestScratchBufferInArena(bytes, buffer_idx); } void* ContextHelper::GetScratchBuffer(TfLiteContext* ctx, int buffer_idx) { - return reinterpret_cast(ctx->impl_) - ->allocator_->GetScratchBuffer(buffer_idx); + ContextHelper* helper = reinterpret_cast(ctx->impl_); + ScratchBufferHandle* handle = helper->scratch_buffer_handles_ + buffer_idx; + return handle->data; } void ContextHelper::ReportOpError(struct TfLiteContext* context, const char* format, ...) { +#ifndef TF_LITE_STRIP_ERROR_STRINGS ContextHelper* helper = static_cast(context->impl_); va_list args; va_start(args, format); TF_LITE_REPORT_ERROR(helper->error_reporter_, format, args); va_end(args); +#endif +} + +TfLiteTensor* ContextHelper::GetTensor(const struct TfLiteContext* context, + int tensor_idx) { + ContextHelper* helper = static_cast(context->impl_); + return helper->allocator_->AllocateTempTfLiteTensor( + helper->model_, helper->eval_tensors_, tensor_idx); +} + +TfLiteEvalTensor* ContextHelper::GetEvalTensor( + const struct TfLiteContext* context, int tensor_idx) { + ContextHelper* helper = reinterpret_cast(context->impl_); + return &helper->eval_tensors_[tensor_idx]; +} + +void ContextHelper::SetTfLiteEvalTensors(TfLiteEvalTensor* eval_tensors) { + eval_tensors_ = eval_tensors; +} + +void ContextHelper::SetScratchBufferHandles( + ScratchBufferHandle* scratch_buffer_handles) { + scratch_buffer_handles_ = scratch_buffer_handles; } } // namespace internal MicroInterpreter::MicroInterpreter(const Model* model, - const OpResolver& op_resolver, + const MicroOpResolver& op_resolver, uint8_t* tensor_arena, size_t tensor_arena_size, - ErrorReporter* error_reporter) + ErrorReporter* error_reporter, + tflite::Profiler* profiler) : model_(model), op_resolver_(op_resolver), error_reporter_(error_reporter), - allocator_(&context_, model_, tensor_arena, tensor_arena_size, - error_reporter_), + allocator_(*MicroAllocator::Create(tensor_arena, tensor_arena_size, + error_reporter)), tensors_allocated_(false), - context_helper_(error_reporter_, &allocator_) { - const flatbuffers::Vector>* subgraphs = - model->subgraphs(); - if (subgraphs->size() != 1) { - TF_LITE_REPORT_ERROR(error_reporter, - "Only 1 subgraph is currently supported.\n"); - initialization_status_ = kTfLiteError; - return; - } - subgraph_ = (*subgraphs)[0]; - tensors_ = subgraph_->tensors(); - operators_ = subgraph_->operators(); + initialization_status_(kTfLiteError), + eval_tensors_(nullptr), + context_helper_(error_reporter_, &allocator_, model), + input_tensor_(nullptr), + output_tensor_(nullptr) { + Init(profiler); +} - context_.impl_ = static_cast(&context_helper_); - context_.ReportError = context_helper_.ReportOpError; - context_.recommended_num_threads = 1; - - // If the system is big endian then convert weights from the flatbuffer from - // little to big endian on startup so that it does not need to be done during - // inference. - // NOTE: This requires that the flatbuffer is held in memory which can be - // modified by this process. - if (!FLATBUFFERS_LITTLEENDIAN) { - for (size_t t = 0; t < tensors_size(); ++t) { - TfLiteTensor* thisTensor = &context_.tensors[t]; - if (thisTensor->allocation_type == kTfLiteMmapRo) - CorrectTensorEndianness(thisTensor); - } - } - - initialization_status_ = kTfLiteOk; +MicroInterpreter::MicroInterpreter(const Model* model, + const MicroOpResolver& op_resolver, + MicroAllocator* allocator, + ErrorReporter* error_reporter, + tflite::Profiler* profiler) + : model_(model), + op_resolver_(op_resolver), + error_reporter_(error_reporter), + allocator_(*allocator), + tensors_allocated_(false), + initialization_status_(kTfLiteError), + eval_tensors_(nullptr), + context_helper_(error_reporter_, &allocator_, model), + input_tensor_(nullptr), + output_tensor_(nullptr) { + Init(profiler); } MicroInterpreter::~MicroInterpreter() { if (node_and_registrations_ != nullptr) { - for (size_t i = 0; i < operators_->size(); ++i) { + for (size_t i = 0; i < subgraph_->operators()->size(); ++i) { TfLiteNode* node = &(node_and_registrations_[i].node); const TfLiteRegistration* registration = node_and_registrations_[i].registration; @@ -125,7 +156,28 @@ MicroInterpreter::~MicroInterpreter() { } } -void MicroInterpreter::CorrectTensorEndianness(TfLiteTensor* tensorCorr) { +void MicroInterpreter::Init(tflite::Profiler* profiler) { + const flatbuffers::Vector>* subgraphs = + model_->subgraphs(); + if (subgraphs->size() != 1) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Only 1 subgraph is currently supported.\n"); + initialization_status_ = kTfLiteError; + return; + } + subgraph_ = (*subgraphs)[0]; + + context_.impl_ = static_cast(&context_helper_); + context_.ReportError = context_helper_.ReportOpError; + context_.GetTensor = context_helper_.GetTensor; + context_.GetEvalTensor = context_helper_.GetEvalTensor; + context_.recommended_num_threads = 1; + context_.profiler = profiler; + + initialization_status_ = kTfLiteOk; +} + +void MicroInterpreter::CorrectTensorEndianness(TfLiteEvalTensor* tensorCorr) { int32_t tensorSize = 1; for (int d = 0; d < tensorCorr->dims->size; ++d) tensorSize *= reinterpret_cast(tensorCorr->dims->data)[d]; @@ -149,6 +201,9 @@ void MicroInterpreter::CorrectTensorEndianness(TfLiteTensor* tensorCorr) { case TfLiteType::kTfLiteComplex64: CorrectTensorDataEndianness(tensorCorr->data.c64, tensorSize); break; + case TfLiteType::kTfLiteComplex128: + CorrectTensorDataEndianness(tensorCorr->data.c128, tensorSize); + break; default: // Do nothing for other data types. break; @@ -163,16 +218,50 @@ void MicroInterpreter::CorrectTensorDataEndianness(T* data, int32_t size) { } TfLiteStatus MicroInterpreter::AllocateTensors() { - TF_LITE_ENSURE_OK(&context_, allocator_.AllocateNodeAndRegistrations( - op_resolver_, &node_and_registrations_)); + if (allocator_.StartModelAllocation(model_, op_resolver_, + &node_and_registrations_, + &eval_tensors_) != kTfLiteOk) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Failed starting model allocation.\n"); + initialization_status_ = kTfLiteError; + return kTfLiteError; + } + + // Update the pointer now that TfLiteEvalTensor allocation has completed on + // the context helper. + // TODO(b/16157777): This call would not be needed if ContextHelper rolled + // into the interpreter. + context_helper_.SetTfLiteEvalTensors(eval_tensors_); + context_.tensors_size = subgraph_->tensors()->size(); + + // If the system is big endian then convert weights from the flatbuffer from + // little to big endian on startup so that it does not need to be done during + // inference. + // NOTE: This requires that the flatbuffer is held in memory which can be + // modified by this process. + if (!FLATBUFFERS_LITTLEENDIAN) { + for (size_t t = 0; t < subgraph_->tensors()->size(); ++t) { + if (auto* buffer = + (*model_->buffers())[subgraph_->tensors()->Get(t)->buffer()]) { + // If we've found a buffer, does it have any data? + if (auto* array = buffer->data()) { + // If it has any data, is the data size larger than zero? + if (array->size()) { + // Update the endianness of the corresponding eval tensor since that + // struct holds the buffer used at inference time. + CorrectTensorEndianness(&eval_tensors_[t]); + } + } + } + } + } // Only allow AllocatePersistentBuffer in Init stage. context_.AllocatePersistentBuffer = context_helper_.AllocatePersistentBuffer; context_.RequestScratchBufferInArena = nullptr; context_.GetScratchBuffer = nullptr; - for (size_t i = 0; i < operators_->size(); ++i) { - context_helper_.SetNodeIndex(i); + for (size_t i = 0; i < subgraph_->operators()->size(); ++i) { auto* node = &(node_and_registrations_[i].node); auto* registration = node_and_registrations_[i].registration; size_t init_data_size; @@ -189,15 +278,12 @@ TfLiteStatus MicroInterpreter::AllocateTensors() { registration->init(&context_, init_data, init_data_size); } } - context_helper_.SetNodeIndex(-1); - // Both AllocatePersistentBuffer and RequestScratchBufferInArena is available - // in Prepare stage. + // Both AllocatePersistentBuffer and RequestScratchBufferInArena is + // available in Prepare stage. context_.RequestScratchBufferInArena = context_helper_.RequestScratchBufferInArena; - for (size_t i = 0; i < operators_->size(); ++i) { - // Set node idx to annotate the lifetime for scratch buffers. - context_helper_.SetNodeIndex(i); + for (size_t i = 0; i < subgraph_->operators()->size(); ++i) { auto* node = &(node_and_registrations_[i].node); auto* registration = node_and_registrations_[i].registration; if (registration->prepare) { @@ -210,8 +296,8 @@ TfLiteStatus MicroInterpreter::AllocateTensors() { return kTfLiteError; } } + allocator_.FinishPrepareNodeAllocations(/*node_id=*/i); } - context_helper_.SetNodeIndex(-1); // Prepare is done, we're ready for Invoke. Memory allocation is no longer // allowed. Kernels can only fetch scratch buffers via GetScratchBuffer. @@ -219,7 +305,14 @@ TfLiteStatus MicroInterpreter::AllocateTensors() { context_.RequestScratchBufferInArena = nullptr; context_.GetScratchBuffer = context_helper_.GetScratchBuffer; - TF_LITE_ENSURE_OK(&context_, allocator_.FinishTensorAllocation()); + TF_LITE_ENSURE_OK(&context_, + allocator_.FinishModelAllocation(model_, eval_tensors_, + &scratch_buffer_handles_)); + // TODO(b/16157777): Remove this when ContextHelper is rolled into this class. + context_helper_.SetScratchBufferHandles(scratch_buffer_handles_); + + TF_LITE_ENSURE_STATUS(ResetVariableTensors()); + tensors_allocated_ = true; return kTfLiteOk; } @@ -237,12 +330,28 @@ TfLiteStatus MicroInterpreter::Invoke() { TF_LITE_ENSURE_OK(&context_, AllocateTensors()); } - for (size_t i = 0; i < operators_->size(); ++i) { + for (size_t i = 0; i < subgraph_->operators()->size(); ++i) { auto* node = &(node_and_registrations_[i].node); auto* registration = node_and_registrations_[i].registration; if (registration->invoke) { - TfLiteStatus invoke_status = registration->invoke(&context_, node); + TfLiteStatus invoke_status; +#ifndef NDEBUG // Omit profiler overhead from release builds. + // The case where profiler == nullptr is handled by + // ScopedOperatorProfile. + tflite::Profiler* profiler = + reinterpret_cast(context_.profiler); + ScopedOperatorProfile scoped_profiler( + profiler, OpNameFromRegistration(registration), i); +#endif + invoke_status = registration->invoke(&context_, node); + + // All TfLiteTensor structs used in the kernel are allocated from temp + // memory in the allocator. This creates a chain of allocations in the + // temp section. The call below resets the chain of allocations to + // prepare for the next call. + allocator_.ResetTempAllocations(); + if (invoke_status == kTfLiteError) { TF_LITE_REPORT_ERROR( error_reporter_, @@ -259,50 +368,82 @@ TfLiteStatus MicroInterpreter::Invoke() { TfLiteTensor* MicroInterpreter::input(size_t index) { const size_t length = inputs_size(); - if ((index < 0) || (index >= length)) { + if (index >= length) { TF_LITE_REPORT_ERROR(error_reporter_, "Input index %d out of range (length is %d)", index, length); return nullptr; } - return &(context_.tensors[inputs().Get(index)]); + if (index != 0) { + TF_LITE_REPORT_ERROR( + error_reporter_, + "Input tensors not at index 0 are allocated from the " + "persistent memory arena. Repeat calls will cause excess " + "allocation!"); + return allocator_.AllocatePersistentTfLiteTensor(model_, eval_tensors_, + inputs().Get(index)); + } + if (input_tensor_ == nullptr) { + input_tensor_ = allocator_.AllocatePersistentTfLiteTensor( + model_, eval_tensors_, inputs().Get(index)); + } + return input_tensor_; } TfLiteTensor* MicroInterpreter::output(size_t index) { const size_t length = outputs_size(); - if ((index < 0) || (index >= length)) { + if (index >= length) { TF_LITE_REPORT_ERROR(error_reporter_, "Output index %d out of range (length is %d)", index, length); return nullptr; } - return &(context_.tensors[outputs().Get(index)]); + if (index != 0) { + TF_LITE_REPORT_ERROR( + error_reporter_, + "Output tensors not at index 0 are allocated from the " + "persistent memory arena. Repeat calls will cause excess " + "allocation!"); + return allocator_.AllocatePersistentTfLiteTensor(model_, eval_tensors_, + outputs().Get(index)); + } + if (output_tensor_ == nullptr) { + // TODO(b/162311891): Drop these allocations when the interpreter supports + // handling buffers from TfLiteEvalTensor. + output_tensor_ = allocator_.AllocatePersistentTfLiteTensor( + model_, eval_tensors_, outputs().Get(index)); + } + return output_tensor_; } TfLiteTensor* MicroInterpreter::tensor(size_t index) { const size_t length = tensors_size(); - if ((index < 0) || (index >= length)) { + if (index >= length) { TF_LITE_REPORT_ERROR(error_reporter_, "Tensor index %d out of range (length is %d)", index, length); return nullptr; } - return &context_.tensors[index]; + return allocator_.AllocatePersistentTfLiteTensor(model_, eval_tensors_, + index); } TfLiteStatus MicroInterpreter::ResetVariableTensors() { - const size_t length = tensors_size(); - for (size_t i = 0; i < length; ++i) { - TfLiteTensor* cur_tensor = tensor(i); - if (cur_tensor->is_variable) { - TfLiteStatus status = tflite::ResetVariableTensor(cur_tensor); - if (status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Failed to reset variable tensor at index: %d", i); - return status; + for (size_t i = 0; i < subgraph_->tensors()->size(); ++i) { + auto* tensor = subgraph_->tensors()->Get(i); + if (tensor->is_variable()) { + size_t buffer_size; + TF_LITE_ENSURE_STATUS( + TfLiteEvalTensorByteLength(&eval_tensors_[i], &buffer_size)); + + int value = 0; + if (tensor->type() == tflite::TensorType_INT8) { + value = tensor->quantization()->zero_point()->Get(0); } + memset(eval_tensors_[i].data.raw, value, buffer_size); } } + return kTfLiteOk; } diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_interpreter.h b/code/lib/tfmicro/tensorflow/lite/micro/micro_interpreter.h index b2046128..31720c8e 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_interpreter.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_interpreter.h @@ -1,4 +1,4 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,13 +15,18 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_MICRO_INTERPRETER_H_ #define TENSORFLOW_LITE_MICRO_MICRO_INTERPRETER_H_ +#include +#include + +#include "flatbuffers/flatbuffers.h" // from @flatbuffers #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" -#include "tensorflow/lite/core/api/op_resolver.h" +#include "tensorflow/lite/core/api/profiler.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/micro/micro_allocator.h" +#include "tensorflow/lite/micro/micro_op_resolver.h" +#include "tensorflow/lite/portable_type_to_tflitetype.h" #include "tensorflow/lite/schema/schema_generated.h" -#include "tensorflow/lite/type_to_tflitetype.h" namespace tflite { @@ -30,47 +35,63 @@ namespace internal { // A helper class to encapsulate the implementation of APIs in Context. // context->impl_ points to an instance of this class. // Check tensorflow/lite/c/common.h for detailed descriptions. +// TODO(b/16157777): Consider rolling this class into MicroInterpreter. class ContextHelper { public: explicit ContextHelper(ErrorReporter* error_reporter, - MicroAllocator* allocator) - : allocator_(allocator), error_reporter_(error_reporter) {} - - static TfLiteStatus AllocatePersistentBuffer(TfLiteContext* ctx, size_t bytes, - void** ptr); + MicroAllocator* allocator, const Model* model); + // Functions that will be assigned to function pointers on TfLiteContext: + static void* AllocatePersistentBuffer(TfLiteContext* ctx, size_t bytes); static TfLiteStatus RequestScratchBufferInArena(TfLiteContext* ctx, size_t bytes, int* buffer_idx); - static void* GetScratchBuffer(TfLiteContext* ctx, int buffer_idx); - static void ReportOpError(struct TfLiteContext* context, const char* format, ...); + static TfLiteTensor* GetTensor(const struct TfLiteContext* context, + int tensor_idx); + static TfLiteEvalTensor* GetEvalTensor(const struct TfLiteContext* context, + int tensor_idx); - void SetNodeIndex(int idx) { current_node_idx_ = idx; } + // Sets the pointer to a list of TfLiteEvalTensor instances. + void SetTfLiteEvalTensors(TfLiteEvalTensor* eval_tensors); + + // Sets the pointer to a list of ScratchBufferHandle instances. + void SetScratchBufferHandles(ScratchBufferHandle* scratch_buffer_handles); private: - MicroAllocator* allocator_; - ErrorReporter* error_reporter_; - int current_node_idx_ = -1; + MicroAllocator* allocator_ = nullptr; + ErrorReporter* error_reporter_ = nullptr; + const Model* model_ = nullptr; + TfLiteEvalTensor* eval_tensors_ = nullptr; + ScratchBufferHandle* scratch_buffer_handles_ = nullptr; }; } // namespace internal class MicroInterpreter { public: - // The lifetime of the model, op resolver, tensor arena, and error reporter - // must be at least as long as that of the interpreter object, since the - // interpreter may need to access them at any time. This means that you should - // usually create them with the same scope as each other, for example having - // them all allocated on the stack as local variables through a top-level - // function. - // The interpreter doesn't do any deallocation of any of the pointed-to - // objects, ownership remains with the caller. - MicroInterpreter(const Model* model, const OpResolver& op_resolver, + // The lifetime of the model, op resolver, tensor arena, error reporter and + // profiler must be at least as long as that of the interpreter object, since + // the interpreter may need to access them at any time. This means that you + // should usually create them with the same scope as each other, for example + // having them all allocated on the stack as local variables through a + // top-level function. The interpreter doesn't do any deallocation of any of + // the pointed-to objects, ownership remains with the caller. + MicroInterpreter(const Model* model, const MicroOpResolver& op_resolver, uint8_t* tensor_arena, size_t tensor_arena_size, - ErrorReporter* error_reporter); + ErrorReporter* error_reporter, + tflite::Profiler* profiler = nullptr); + + // Create an interpreter instance using an existing MicroAllocator instance. + // This constructor should be used when creating an allocator that needs to + // have allocation handled in more than one interpreter or for recording + // allocations inside the interpreter. The lifetime of the allocator must be + // as long as that of the interpreter object. + MicroInterpreter(const Model* model, const MicroOpResolver& op_resolver, + MicroAllocator* allocator, ErrorReporter* error_reporter, + tflite::Profiler* profiler = nullptr); ~MicroInterpreter(); @@ -132,7 +153,7 @@ class MicroInterpreter { TfLiteStatus initialization_status() const { return initialization_status_; } - size_t operators_size() const { return operators_->size(); } + size_t operators_size() const { return subgraph_->operators()->size(); } // For debugging only. const NodeAndRegistration node_and_registration(int node_index) const { @@ -147,8 +168,16 @@ class MicroInterpreter { // arena_used_bytes() + 16. size_t arena_used_bytes() const { return allocator_.used_bytes(); } + protected: + const MicroAllocator& allocator() const { return allocator_; } + const TfLiteContext& context() const { return context_; } + private: - void CorrectTensorEndianness(TfLiteTensor* tensorCorr); + // TODO(b/158263161): Consider switching to Create() function to enable better + // error reporting during initialization. + void Init(tflite::Profiler* profiler); + + void CorrectTensorEndianness(TfLiteEvalTensor* tensorCorr); template void CorrectTensorDataEndianness(T* data, int32_t size); @@ -156,18 +185,25 @@ class MicroInterpreter { NodeAndRegistration* node_and_registrations_ = nullptr; const Model* model_; - const OpResolver& op_resolver_; + const MicroOpResolver& op_resolver_; ErrorReporter* error_reporter_; TfLiteContext context_ = {}; - MicroAllocator allocator_; + MicroAllocator& allocator_; bool tensors_allocated_; TfLiteStatus initialization_status_; - const flatbuffers::Vector>* tensors_; - const flatbuffers::Vector>* operators_; - const SubGraph* subgraph_; + const SubGraph* subgraph_ = nullptr; + TfLiteEvalTensor* eval_tensors_ = nullptr; + ScratchBufferHandle* scratch_buffer_handles_ = nullptr; + + // TODO(b/16157777): Drop this reference: internal::ContextHelper context_helper_; + + // TODO(b/162311891): Clean these pointers up when this class supports buffers + // from TfLiteEvalTensor. + TfLiteTensor* input_tensor_; + TfLiteTensor* output_tensor_; }; } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_mutable_op_resolver.h b/code/lib/tfmicro/tensorflow/lite/micro/micro_mutable_op_resolver.h index ac304352..0175c8db 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_mutable_op_resolver.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_mutable_op_resolver.h @@ -1,4 +1,4 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,109 +15,454 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_MICRO_MUTABLE_OP_RESOLVER_H_ #define TENSORFLOW_LITE_MICRO_MICRO_MUTABLE_OP_RESOLVER_H_ -#include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/core/api/op_resolver.h" -#include "tensorflow/lite/micro/compatibility.h" -#include "tensorflow/lite/schema/schema_generated.h" +#include +#include -#ifndef TFLITE_REGISTRATIONS_MAX -#define TFLITE_REGISTRATIONS_MAX (128) -#endif +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/core/api/flatbuffer_conversions.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/compatibility.h" +#include "tensorflow/lite/micro/kernels/fully_connected.h" +#include "tensorflow/lite/micro/kernels/micro_ops.h" +#include "tensorflow/lite/micro/micro_op_resolver.h" +#include "tensorflow/lite/schema/schema_generated.h" namespace tflite { -// Op versions discussed in this file are enumerated here: -// tensorflow/lite/tools/versioning/op_version.cc - -inline int MicroOpResolverAnyVersion() { return 0; } - -template -class MicroOpResolver : public OpResolver { +template +class MicroMutableOpResolver : public MicroOpResolver { public: - const TfLiteRegistration* FindOp(tflite::BuiltinOperator op, - int version) const override { + explicit MicroMutableOpResolver(ErrorReporter* error_reporter = nullptr) + : error_reporter_(error_reporter) {} + + const TfLiteRegistration* FindOp(tflite::BuiltinOperator op) const override { + if (op == BuiltinOperator_CUSTOM) return nullptr; + for (unsigned int i = 0; i < registrations_len_; ++i) { const TfLiteRegistration& registration = registrations_[i]; - if ((registration.builtin_code == op) && - (registration.version == MicroOpResolverAnyVersion() || - registration.version == version)) { + if (registration.builtin_code == op) { return ®istration; } } return nullptr; } - const TfLiteRegistration* FindOp(const char* op, int version) const override { + const TfLiteRegistration* FindOp(const char* op) const override { for (unsigned int i = 0; i < registrations_len_; ++i) { const TfLiteRegistration& registration = registrations_[i]; if ((registration.builtin_code == BuiltinOperator_CUSTOM) && - (strcmp(registration.custom_name, op) == 0) && - (registration.version == MicroOpResolverAnyVersion() || - registration.version == version)) { + (strcmp(registration.custom_name, op) == 0)) { return ®istration; } } return nullptr; } - void AddBuiltin(tflite::BuiltinOperator op, TfLiteRegistration* registration, - int version = 1) { - if (registrations_len_ >= tOpCount) { - // TODO(b/147748244) - Add error reporting hooks so we can report this! - return; + MicroOpResolver::BuiltinParseFunction GetOpDataParser( + BuiltinOperator op) const override { + TFLITE_DCHECK(num_buitin_ops_ <= tOpCount); + for (unsigned int i = 0; i < num_buitin_ops_; ++i) { + if (builtin_codes_[i] == op) return builtin_parsers_[i]; } - TfLiteRegistration* new_registration = ®istrations_[registrations_len_]; - registrations_len_ += 1; - - *new_registration = *registration; - new_registration->builtin_code = op; - new_registration->version = version; + return nullptr; } - void AddBuiltin(tflite::BuiltinOperator op, TfLiteRegistration* registration, - int min_version, int max_version) { - for (int version = min_version; version <= max_version; ++version) { - AddBuiltin(op, registration, version); - } - } - - void AddCustom(const char* name, TfLiteRegistration* registration, - int version = 1) { + // Registers a Custom Operator with the MicroOpResolver. + // + // Only the first call for a given name will be successful. i.e. if this + // function is called again for a previously added Custom Operator, the + // MicroOpResolver will be unchanged and this function will return + // kTfLiteError. + TfLiteStatus AddCustom(const char* name, TfLiteRegistration* registration) { if (registrations_len_ >= tOpCount) { - // TODO(b/147748244) - Add error reporting hooks so we can report this! - return; + if (error_reporter_) { + TF_LITE_REPORT_ERROR( + error_reporter_, + "Couldn't register custom op '%s', resolver size is too small (%d)", + name, tOpCount); + } + return kTfLiteError; } + + if (FindOp(name) != nullptr) { + if (error_reporter_ != nullptr) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Calling AddCustom for the same op more than once " + "is not supported (Op: %s).", + name); + } + return kTfLiteError; + } + TfLiteRegistration* new_registration = ®istrations_[registrations_len_]; registrations_len_ += 1; *new_registration = *registration; new_registration->builtin_code = BuiltinOperator_CUSTOM; new_registration->custom_name = name; - new_registration->version = version; + return kTfLiteOk; } - void AddCustom(const char* name, TfLiteRegistration* registration, - int min_version, int max_version) { - for (int version = min_version; version <= max_version; ++version) { - AddCustom(name, registration, version); - } + // The Add* functions below add the various Builtin operators to the + // MicroMutableOpResolver object. + + TfLiteStatus AddAbs() { + return AddBuiltin(BuiltinOperator_ABS, tflite::ops::micro::Register_ABS(), + ParseAbs); + } + + TfLiteStatus AddAdd() { + return AddBuiltin(BuiltinOperator_ADD, tflite::ops::micro::Register_ADD(), + ParseAdd); + } + + TfLiteStatus AddArgMax() { + return AddBuiltin(BuiltinOperator_ARG_MAX, + tflite::ops::micro::Register_ARG_MAX(), ParseArgMax); + } + + TfLiteStatus AddArgMin() { + return AddBuiltin(BuiltinOperator_ARG_MIN, + tflite::ops::micro::Register_ARG_MIN(), ParseArgMin); + } + + TfLiteStatus AddAveragePool2D() { + return AddBuiltin(BuiltinOperator_AVERAGE_POOL_2D, + tflite::ops::micro::Register_AVERAGE_POOL_2D(), + ParsePool); + } + + TfLiteStatus AddCeil() { + return AddBuiltin(BuiltinOperator_CEIL, tflite::ops::micro::Register_CEIL(), + ParseCeil); + } + + TfLiteStatus AddCircularBuffer() { + return AddCustom("CIRCULAR_BUFFER", + tflite::ops::micro::Register_CIRCULAR_BUFFER()); + } + + TfLiteStatus AddConcatenation() { + return AddBuiltin(BuiltinOperator_CONCATENATION, + tflite::ops::micro::Register_CONCATENATION(), + ParseConcatenation); + } + + TfLiteStatus AddConv2D() { + return AddBuiltin(BuiltinOperator_CONV_2D, Register_CONV_2D(), ParseConv2D); + } + + TfLiteStatus AddCos() { + return AddBuiltin(BuiltinOperator_COS, tflite::ops::micro::Register_COS(), + ParseCos); + } + + TfLiteStatus AddDepthwiseConv2D() { + return AddBuiltin(BuiltinOperator_DEPTHWISE_CONV_2D, + Register_DEPTHWISE_CONV_2D(), ParseDepthwiseConv2D); + } + + TfLiteStatus AddDequantize() { + return AddBuiltin(BuiltinOperator_DEQUANTIZE, + tflite::ops::micro::Register_DEQUANTIZE(), + ParseDequantize); + } + + TfLiteStatus AddEqual() { + return AddBuiltin(BuiltinOperator_EQUAL, + tflite::ops::micro::Register_EQUAL(), ParseEqual); + } + + TfLiteStatus AddFloor() { + return AddBuiltin(BuiltinOperator_FLOOR, + tflite::ops::micro::Register_FLOOR(), ParseFloor); + } + + TfLiteStatus AddFullyConnected( + const TfLiteRegistration& registration = Register_FULLY_CONNECTED()) { + return AddBuiltin(BuiltinOperator_FULLY_CONNECTED, registration, + ParseFullyConnected); + } + + TfLiteStatus AddGreater() { + return AddBuiltin(BuiltinOperator_GREATER, + tflite::ops::micro::Register_GREATER(), ParseGreater); + } + + TfLiteStatus AddGreaterEqual() { + return AddBuiltin(BuiltinOperator_GREATER_EQUAL, + tflite::ops::micro::Register_GREATER_EQUAL(), + ParseGreaterEqual); + } + + TfLiteStatus AddHardSwish() { + return AddBuiltin(BuiltinOperator_HARD_SWISH, + tflite::ops::micro::Register_HARD_SWISH(), + ParseHardSwish); + } + + TfLiteStatus AddL2Normalization() { + return AddBuiltin(BuiltinOperator_L2_NORMALIZATION, + tflite::ops::micro::Register_L2_NORMALIZATION(), + ParseL2Normalization); + } + + TfLiteStatus AddLess() { + return AddBuiltin(BuiltinOperator_LESS, tflite::ops::micro::Register_LESS(), + ParseLess); + } + + TfLiteStatus AddLessEqual() { + return AddBuiltin(BuiltinOperator_LESS_EQUAL, + tflite::ops::micro::Register_LESS_EQUAL(), + ParseLessEqual); + } + + TfLiteStatus AddLog() { + return AddBuiltin(BuiltinOperator_LOG, tflite::ops::micro::Register_LOG(), + ParseLog); + } + + TfLiteStatus AddLogicalAnd() { + return AddBuiltin(BuiltinOperator_LOGICAL_AND, + tflite::ops::micro::Register_LOGICAL_AND(), + ParseLogicalAnd); + } + + TfLiteStatus AddLogicalNot() { + return AddBuiltin(BuiltinOperator_LOGICAL_NOT, + tflite::ops::micro::Register_LOGICAL_NOT(), + ParseLogicalNot); + } + + TfLiteStatus AddLogicalOr() { + return AddBuiltin(BuiltinOperator_LOGICAL_OR, + tflite::ops::micro::Register_LOGICAL_OR(), + ParseLogicalOr); + } + + TfLiteStatus AddLogistic() { + return AddBuiltin(BuiltinOperator_LOGISTIC, + tflite::ops::micro::Register_LOGISTIC(), ParseLogistic); + } + + TfLiteStatus AddMaximum() { + return AddBuiltin(BuiltinOperator_MAXIMUM, + tflite::ops::micro::Register_MAXIMUM(), ParseMaximum); + } + + TfLiteStatus AddMaxPool2D() { + return AddBuiltin(BuiltinOperator_MAX_POOL_2D, + tflite::ops::micro::Register_MAX_POOL_2D(), ParsePool); + } + + TfLiteStatus AddMean() { + return AddBuiltin(BuiltinOperator_MEAN, tflite::ops::micro::Register_MEAN(), + ParseReducer); + } + + TfLiteStatus AddMinimum() { + return AddBuiltin(BuiltinOperator_MINIMUM, + tflite::ops::micro::Register_MINIMUM(), ParseMinimum); + } + + TfLiteStatus AddMul() { + return AddBuiltin(BuiltinOperator_MUL, tflite::ops::micro::Register_MUL(), + ParseMul); + } + + TfLiteStatus AddNeg() { + return AddBuiltin(BuiltinOperator_NEG, tflite::ops::micro::Register_NEG(), + ParseNeg); + } + + TfLiteStatus AddNotEqual() { + return AddBuiltin(BuiltinOperator_NOT_EQUAL, + tflite::ops::micro::Register_NOT_EQUAL(), ParseNotEqual); + } + + TfLiteStatus AddPack() { + return AddBuiltin(BuiltinOperator_PACK, tflite::ops::micro::Register_PACK(), + ParsePack); + } + + TfLiteStatus AddPad() { + return AddBuiltin(BuiltinOperator_PAD, tflite::ops::micro::Register_PAD(), + ParsePad); + } + + TfLiteStatus AddPadV2() { + return AddBuiltin(BuiltinOperator_PADV2, + tflite::ops::micro::Register_PADV2(), ParsePadV2); + } + + TfLiteStatus AddPrelu() { + return AddBuiltin(BuiltinOperator_PRELU, + tflite::ops::micro::Register_PRELU(), ParsePrelu); + } + + TfLiteStatus AddQuantize() { + return AddBuiltin(BuiltinOperator_QUANTIZE, Register_QUANTIZE(), + ParseQuantize); + } + + TfLiteStatus AddReduceMax() { + return AddBuiltin(BuiltinOperator_REDUCE_MAX, + tflite::ops::micro::Register_REDUCE_MAX(), ParseReducer); + } + + TfLiteStatus AddRelu() { + return AddBuiltin(BuiltinOperator_RELU, tflite::ops::micro::Register_RELU(), + ParseRelu); + } + + TfLiteStatus AddRelu6() { + return AddBuiltin(BuiltinOperator_RELU6, + tflite::ops::micro::Register_RELU6(), ParseRelu6); + } + + TfLiteStatus AddReshape() { + return AddBuiltin(BuiltinOperator_RESHAPE, + tflite::ops::micro::Register_RESHAPE(), ParseReshape); + } + + TfLiteStatus AddResizeNearestNeighbor() { + return AddBuiltin(BuiltinOperator_RESIZE_NEAREST_NEIGHBOR, + tflite::ops::micro::Register_RESIZE_NEAREST_NEIGHBOR(), + ParseResizeNearestNeighbor); + } + + TfLiteStatus AddRound() { + return AddBuiltin(BuiltinOperator_ROUND, + tflite::ops::micro::Register_ROUND(), ParseRound); + } + + TfLiteStatus AddRsqrt() { + return AddBuiltin(BuiltinOperator_RSQRT, + tflite::ops::micro::Register_RSQRT(), ParseRsqrt); + } + + TfLiteStatus AddShape() { + return AddBuiltin(BuiltinOperator_SHAPE, Register_SHAPE(), ParseShape); + } + + TfLiteStatus AddSin() { + return AddBuiltin(BuiltinOperator_SIN, tflite::ops::micro::Register_SIN(), + ParseSin); + } + + TfLiteStatus AddSoftmax() { + return AddBuiltin(BuiltinOperator_SOFTMAX, Register_SOFTMAX(), + ParseSoftmax); + } + + TfLiteStatus AddSplit() { + return AddBuiltin(BuiltinOperator_SPLIT, + tflite::ops::micro::Register_SPLIT(), ParseSplit); + } + + TfLiteStatus AddSplitV() { + return AddBuiltin(BuiltinOperator_SPLIT_V, + tflite::ops::micro::Register_SPLIT_V(), ParseSplitV); + } + + TfLiteStatus AddSqrt() { + return AddBuiltin(BuiltinOperator_SQRT, tflite::ops::micro::Register_SQRT(), + ParseSqrt); + } + + TfLiteStatus AddSquare() { + return AddBuiltin(BuiltinOperator_SQUARE, + tflite::ops::micro::Register_SQUARE(), ParseSquare); + } + + TfLiteStatus AddStridedSlice() { + return AddBuiltin(BuiltinOperator_STRIDED_SLICE, + tflite::ops::micro::Register_STRIDED_SLICE(), + ParseStridedSlice); + } + + TfLiteStatus AddSub() { + return AddBuiltin(BuiltinOperator_SUB, tflite::ops::micro::Register_SUB(), + ParseSub); + } + + TfLiteStatus AddSvdf() { + return AddBuiltin(BuiltinOperator_SVDF, Register_SVDF(), ParseSvdf); + } + + TfLiteStatus AddTanh() { + return AddBuiltin(BuiltinOperator_TANH, tflite::ops::micro::Register_TANH(), + ParseTanh); + } + + TfLiteStatus AddUnpack() { + return AddBuiltin(BuiltinOperator_UNPACK, + tflite::ops::micro::Register_UNPACK(), ParseUnpack); } unsigned int GetRegistrationLength() { return registrations_len_; } private: + TF_LITE_REMOVE_VIRTUAL_DELETE + + TfLiteStatus AddBuiltin(tflite::BuiltinOperator op, + const TfLiteRegistration& registration, + MicroOpResolver::BuiltinParseFunction parser) { + if (op == BuiltinOperator_CUSTOM) { + if (error_reporter_ != nullptr) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Invalid parameter BuiltinOperator_CUSTOM to the " + "AddBuiltin function."); + } + return kTfLiteError; + } + + if (FindOp(op) != nullptr) { + if (error_reporter_ != nullptr) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Calling AddBuiltin with the same op more than " + "once is not supported (Op: #%d).", + op); + } + return kTfLiteError; + } + + if (registrations_len_ >= tOpCount) { + if (error_reporter_) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Couldn't register builtin op #%d, resolver size " + "is too small (%d).", + op, tOpCount); + } + return kTfLiteError; + } + + registrations_[registrations_len_] = registration; + // Strictly speaking, the builtin_code is not necessary for TFLM but filling + // it in regardless. + registrations_[registrations_len_].builtin_code = op; + registrations_len_++; + + builtin_codes_[num_buitin_ops_] = op; + builtin_parsers_[num_buitin_ops_] = parser; + num_buitin_ops_++; + + return kTfLiteOk; + } + TfLiteRegistration registrations_[tOpCount]; unsigned int registrations_len_ = 0; - TF_LITE_REMOVE_VIRTUAL_DELETE -}; + // Arrays (and counter) to store the builtin codes and their corresponding + // parse functions as these are registered with the Op Resolver. + BuiltinOperator builtin_codes_[tOpCount]; + MicroOpResolver::BuiltinParseFunction builtin_parsers_[tOpCount]; + unsigned int num_buitin_ops_ = 0; -// TODO(b/147854028): Consider switching all uses of MicroMutableOpResolver to -// MicroOpResolver. -class MicroMutableOpResolver - : public MicroOpResolver { - private: - TF_LITE_REMOVE_VIRTUAL_DELETE + ErrorReporter* error_reporter_; }; }; // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_op_resolver.h b/code/lib/tfmicro/tensorflow/lite/micro/micro_op_resolver.h new file mode 100644 index 00000000..757b6b89 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_op_resolver.h @@ -0,0 +1,73 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_MICRO_MICRO_OP_RESOLVER_H_ +#define TENSORFLOW_LITE_MICRO_MICRO_OP_RESOLVER_H_ + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/core/api/flatbuffer_conversions.h" +#include "tensorflow/lite/core/api/op_resolver.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { + +// This is an interface for the OpResolver for TFLiteMicro. The differences from +// the TFLite OpResolver base class are to: +// * explicitly remove support for Op versions +// * allow for finer grained registration of the Builtin Ops to reduce code +// size for TFLiteMicro. +// +// We need an interface class instead of directly using MicroMutableOpResolver +// because MicroMutableOpResolver is a class template with the number of +// registered Ops as the template parameter. +class MicroOpResolver : public OpResolver { + public: + typedef TfLiteStatus (*BuiltinParseFunction)(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + + // Returns the Op registration struct corresponding to the enum code from the + // flatbuffer schema. Returns nullptr if the op is not found or if op == + // BuiltinOperator_CUSTOM. + virtual const TfLiteRegistration* FindOp(BuiltinOperator op) const = 0; + + // Returns the Op registration struct corresponding to the custom operator by + // name. + virtual const TfLiteRegistration* FindOp(const char* op) const = 0; + + // This implementation exists for compatibility with the OpResolver base class + // and disregards the version parameter. + const TfLiteRegistration* FindOp(BuiltinOperator op, + int version) const final { + return FindOp(op); + } + + // This implementation exists for compatibility with the OpResolver base class + // and disregards the version parameter. + const TfLiteRegistration* FindOp(const char* op, int version) const final { + return FindOp(op); + } + + // Returns the operator specific parsing function for the OpData for a + // BuiltinOperator (if registered), else nullptr. + virtual BuiltinParseFunction GetOpDataParser(BuiltinOperator op) const = 0; + + ~MicroOpResolver() override {} +}; + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_MICRO_OP_RESOLVER_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_optional_debug_tools.cc b/code/lib/tfmicro/tensorflow/lite/micro/micro_optional_debug_tools.cc deleted file mode 100644 index 70f16c78..00000000 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_optional_debug_tools.cc +++ /dev/null @@ -1,144 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/lite/micro/micro_optional_debug_tools.h" - -// `cinttypes` requires `__STDC_FORMAT_MACROS` to be defined to expose `PRId32`. -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -#include - -#include "tensorflow/lite/schema/schema_generated.h" -namespace tflite { -namespace { - -std::vector flatbuffersVector2StdVector( - const flatbuffers::Vector& fVector) { - std::vector stdVector; - stdVector.reserve(fVector.size()); - for (size_t i = 0; i < fVector.size(); i++) { - stdVector.push_back(fVector.Get(i)); - } - return stdVector; -} - -void PrintIntVector(const std::vector& v) { - for (const auto& it : v) { - printf(" %d", it); - } - printf("\n"); -} - -void PrintTfLiteIntVector(const TfLiteIntArray* v) { - if (!v) { - printf(" (null)\n"); - return; - } - for (int k = 0; k < v->size; k++) { - printf(" %d", v->data[k]); - } - printf("\n"); -} - -const char* TensorTypeName(TfLiteType type) { - switch (type) { - case kTfLiteNoType: - return "kTfLiteNoType"; - case kTfLiteFloat32: - return "kTfLiteFloat32"; - case kTfLiteInt32: - return "kTfLiteInt32"; - case kTfLiteUInt8: - return "kTfLiteUInt8"; - case kTfLiteInt8: - return "kTfLiteInt8"; - case kTfLiteInt64: - return "kTfLiteInt64"; - case kTfLiteString: - return "kTfLiteString"; - case kTfLiteBool: - return "kTfLiteBool"; - case kTfLiteInt16: - return "kTfLiteInt16"; - case kTfLiteComplex64: - return "kTfLiteComplex64"; - case kTfLiteFloat16: - return "kTfLiteFloat16"; - case kTfLiteFloat64: - return "kTfLiteFloat64"; - } - return "(invalid)"; -} - -const char* AllocTypeName(TfLiteAllocationType type) { - switch (type) { - case kTfLiteMemNone: - return "kTfLiteMemNone"; - case kTfLiteMmapRo: - return "kTfLiteMmapRo"; - case kTfLiteDynamic: - return "kTfLiteDynamic"; - case kTfLiteArenaRw: - return "kTfLiteArenaRw"; - case kTfLiteArenaRwPersistent: - return "kTfLiteArenaRwPersistent"; - } - return "(invalid)"; -} -} // namespace - -// Prints a dump of what tensors and what nodes are in the interpreter. -void PrintInterpreterState(MicroInterpreter* interpreter) { - printf("Interpreter has %zu tensors and %zu nodes\n", - interpreter->tensors_size(), interpreter->operators_size()); - printf("Inputs:"); - PrintIntVector(flatbuffersVector2StdVector(interpreter->inputs())); - printf("Outputs:"); - PrintIntVector(flatbuffersVector2StdVector(interpreter->outputs())); - printf("\n"); - - for (size_t tensor_index = 0; tensor_index < interpreter->tensors_size(); - tensor_index++) { - TfLiteTensor* tensor = interpreter->tensor(static_cast(tensor_index)); - printf("Tensor %3zu %-20s %10s %15s %10zu bytes (%4.1f MB) ", tensor_index, - tensor->name, TensorTypeName(tensor->type), - AllocTypeName(tensor->allocation_type), tensor->bytes, - static_cast(tensor->bytes / (1 << 20))); - PrintTfLiteIntVector(tensor->dims); - } - printf("\n"); - - for (size_t node_index = 0; node_index < interpreter->operators_size(); - node_index++) { - const NodeAndRegistration node_and_reg = - interpreter->node_and_registration(static_cast(node_index)); - const TfLiteNode& node = node_and_reg.node; - const TfLiteRegistration* reg = node_and_reg.registration; - if (reg->custom_name != nullptr) { - printf("Node %3zu Operator Custom Name %s\n", node_index, - reg->custom_name); - } else { - printf("Node %3zu Operator Builtin Code %3" PRId32 " %s\n", node_index, - reg->builtin_code, EnumNamesBuiltinOperator()[reg->builtin_code]); - } - printf(" Inputs:"); - PrintTfLiteIntVector(node.inputs); - printf(" Outputs:"); - PrintTfLiteIntVector(node.outputs); - } -} - -} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_profiler.cc b/code/lib/tfmicro/tensorflow/lite/micro/micro_profiler.cc new file mode 100644 index 00000000..83fb9f64 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_profiler.cc @@ -0,0 +1,42 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/micro_profiler.h" + +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/micro_time.h" + +namespace tflite { + +MicroProfiler::MicroProfiler(tflite::ErrorReporter* reporter) + : reporter_(reporter) {} + +uint32_t MicroProfiler::BeginEvent(const char* tag, EventType event_type, + int64_t event_metadata1, + int64_t event_metadata2) { + start_time_ = GetCurrentTimeTicks(); + TFLITE_DCHECK(tag != nullptr); + event_tag_ = tag; + return 0; +} + +void MicroProfiler::EndEvent(uint32_t event_handle) { +#ifndef TF_LITE_STRIP_ERROR_STRINGS + int32_t end_time = GetCurrentTimeTicks(); + TF_LITE_REPORT_ERROR(reporter_, "%s took %d cycles\n", event_tag_, + end_time - start_time_); +#endif +} +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_profiler.h b/code/lib/tfmicro/tensorflow/lite/micro/micro_profiler.h new file mode 100644 index 00000000..a3144b3a --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_profiler.h @@ -0,0 +1,71 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_MICRO_PROFILER_H_ +#define TENSORFLOW_LITE_MICRO_MICRO_PROFILER_H_ + +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/core/api/profiler.h" +#include "tensorflow/lite/micro/compatibility.h" + +namespace tflite { + +// MicroProfiler creates a common way to gain fine-grained insight into runtime +// performance. Bottleck operators can be identified along with slow code +// sections. This can be used in conjunction with running the relevant micro +// benchmark to evaluate end-to-end performance. +// +// Usage example: +// MicroProfiler profiler(error_reporter); +// { +// ScopedProfile scoped_profile(profiler, tag); +// work_to_profile(); +// } +// +// This will call the following methods in order: +// int event_handle = profiler->BeginEvent(op_name, EventType::DEFAULT, 0) +// work_to_profile(); +// profiler->EndEvent(event_handle) +class MicroProfiler : public tflite::Profiler { + public: + explicit MicroProfiler(tflite::ErrorReporter* reporter); + ~MicroProfiler() override = default; + + // AddEvent is unused for Tf Micro. + void AddEvent(const char* tag, EventType event_type, uint64_t start, + uint64_t end, int64_t event_metadata1, + int64_t event_metadata2) override{}; + + // BeginEvent followed by code followed by EndEvent will profile the code + // enclosed. Multiple concurrent events are unsupported, so the return value + // is always 0. Event_metadata1 and event_metadata2 are unused. The tag + // pointer must be valid until EndEvent is called. + uint32_t BeginEvent(const char* tag, EventType event_type, + int64_t event_metadata1, + int64_t event_metadata2) override; + + // Event_handle is ignored since TF Micro does not support concurrent events. + void EndEvent(uint32_t event_handle) override; + + private: + tflite::ErrorReporter* reporter_; + int32_t start_time_; + const char* event_tag_; + TF_LITE_REMOVE_VIRTUAL_DELETE +}; + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_MICRO_PROFILER_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_string.cc b/code/lib/tfmicro/tensorflow/lite/micro/micro_string.cc index 9952565e..ad769f69 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_string.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_string.cc @@ -23,6 +23,7 @@ limitations under the License. #include #include +#include namespace { @@ -125,7 +126,8 @@ char* FastFloatToBufferLeft(float f, char* buffer) { const int32_t exponent_shift = 23; const int32_t exponent_bias = 127; const uint32_t fraction_mask = 0x007fffff; - const uint32_t u = *reinterpret_cast(&f); + uint32_t u; + memcpy(&u, &f, sizeof(int32_t)); const int32_t exponent = ((u & exponent_mask) >> exponent_shift) - exponent_bias; const uint32_t fraction = (u & fraction_mask); @@ -163,7 +165,49 @@ char* FastFloatToBufferLeft(float f, char* buffer) { *current = '.'; current += 1; *current = 0; + + // Prepend leading zeros to fill in all 7 bytes of the fraction. Truncate + // zeros off the end of the fraction. Every fractional value takes 7 bytes. + // For example, 2500 would be written into the buffer as 0002500 since it + // represents .00025. + constexpr int kMaxFractionalDigits = 7; + + // Abort early if there is not enough space in the buffer. + if (current_end - current <= kMaxFractionalDigits) { + return current; + } + + // Pre-fill buffer with zeros to ensure zero-truncation works properly. + for (int i = 1; i < kMaxFractionalDigits; i++) { + *(current + i) = '0'; + } + + // Track how large the fraction is to add leading zeros. + char* previous = current; current = StrCatUInt32(current, (current_end - current), scaled_fraction, 10); + int fraction_digits = current - previous; + int leading_zeros = kMaxFractionalDigits - fraction_digits; + + // Overwrite the null terminator from StrCatUInt32 to ensure zero-trunctaion + // works properly. + *current = '0'; + + // Shift fraction values and prepend zeros if necessary. + if (leading_zeros != 0) { + for (int i = 0; i < fraction_digits; i++) { + current--; + *(current + leading_zeros) = *current; + *current = '0'; + } + current += kMaxFractionalDigits; + } + + // Truncate trailing zeros for cleaner logs. Ensure we leave at least one + // fractional character for the case when scaled_fraction is 0. + while (*(current - 1) == '0' && (current - 1) > previous) { + current--; + } + *current = 0; current = StrCatStr(current, (current_end - current), "*2^"); current = StrCatInt32(current, (current_end - current), exponent); return current; diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_utils.cc b/code/lib/tfmicro/tensorflow/lite/micro/micro_utils.cc index ff885fa0..96152364 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_utils.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_utils.cc @@ -15,34 +15,15 @@ limitations under the License. #include "tensorflow/lite/micro/micro_utils.h" -#include -#include -#include +#include +#include +#include #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/op_macros.h" namespace tflite { -namespace { - -static const uint8_t kAsymmetricUInt8Min = 0; -static const uint8_t kAsymmetricUInt8Max = UINT8_MAX; -static const uint8_t kSymmetricUInt8Min = 1; -static const uint8_t kSymmetricUInt8Max = UINT8_MAX; -static const int8_t kAsymmetricInt8Min = INT8_MIN; -static const int8_t kAsymmetricInt8Max = INT8_MAX; -static const int kSymmetricInt8Scale = kAsymmetricInt8Max; - -static const int16_t kAsymmetricInt16Min = INT16_MIN; -static const int16_t kAsymmetricInt16Max = INT16_MAX; -static const int kSymmetricInt16Scale = kAsymmetricInt16Max; - -static const int32_t kAsymmetricInt32Max = INT32_MAX; -static const int kSymmetricInt32Scale = kAsymmetricInt32Max; - -} // namespace - int ElementCount(const TfLiteIntArray& dims) { int result = 1; for (int i = 0; i < dims.size; ++i) { @@ -51,109 +32,6 @@ int ElementCount(const TfLiteIntArray& dims) { return result; } -// Converts a float value into an unsigned eight-bit quantized value. -uint8_t FloatToAsymmetricQuantizedUInt8(const float value, const float scale, - const int zero_point) { - int32_t result = round(value / scale) + zero_point; - if (result < kAsymmetricUInt8Min) { - result = kAsymmetricUInt8Min; - } - if (result > kAsymmetricUInt8Max) { - result = kAsymmetricUInt8Max; - } - return result; -} - -uint8_t FloatToSymmetricQuantizedUInt8(const float value, const float scale) { - int32_t result = round(value / scale); - if (result < kSymmetricUInt8Min) { - result = kSymmetricUInt8Min; - } - if (result > kSymmetricUInt8Max) { - result = kSymmetricUInt8Max; - } - return result; -} - -int8_t FloatToAsymmetricQuantizedInt8(const float value, const float scale, - const int zero_point) { - int32_t result = round(value / scale) + zero_point; - if (result < kAsymmetricInt8Min) { - result = kAsymmetricInt8Min; - } - if (result > kAsymmetricInt8Max) { - result = kAsymmetricInt8Max; - } - return result; -} - -int16_t FloatToAsymmetricQuantizedInt16(const float value, const float scale, - const int zero_point) { - int32_t result = round(value / scale) + zero_point; - if (result < kAsymmetricInt16Min) { - result = kAsymmetricInt16Min; - } - if (result > kAsymmetricInt16Max) { - result = kAsymmetricInt16Max; - } - return result; -} - -int8_t FloatToSymmetricQuantizedInt8(const float value, const float scale) { - return FloatToAsymmetricQuantizedInt8(value, scale, 0.0f); -} - -int32_t FloatToSymmetricQuantizedInt32(const float value, const float scale) { - float quantized = round(value / scale); - if (static_cast(quantized) > INT_MAX) { - quantized = static_cast(INT_MAX); - } else if (quantized < INT_MIN) { - quantized = static_cast INT_MIN; - } - - return static_cast(quantized); -} - -void AsymmetricQuantize(const float* input, int8_t* output, int num_elements, - float scale, int zero_point) { - for (int i = 0; i < num_elements; i++) { - output[i] = FloatToAsymmetricQuantizedInt8(input[i], scale, zero_point); - } -} - -void AsymmetricQuantize(const float* input, uint8_t* output, int num_elements, - float scale, int zero_point) { - for (int i = 0; i < num_elements; i++) { - output[i] = FloatToAsymmetricQuantizedUInt8(input[i], scale, zero_point); - } -} - -void AsymmetricQuantize(const float* input, int16_t* output, int num_elements, - float scale, int zero_point) { - for (int i = 0; i < num_elements; i++) { - output[i] = FloatToAsymmetricQuantizedInt16(input[i], scale, zero_point); - } -} - -void SymmetricQuantize(const float* input, int32_t* output, int num_elements, - float scale) { - for (int i = 0; i < num_elements; i++) { - output[i] = FloatToSymmetricQuantizedInt32(input[i], scale); - } -} - -void SymmetricPerChannelQuantize(const float* input, int32_t* output, - int num_elements, int num_channels, - float* scales) { - int elements_per_channel = num_elements / num_channels; - for (int i = 0; i < num_channels; i++) { - for (int j = 0; j < elements_per_channel; j++) { - output[i * elements_per_channel + j] = FloatToSymmetricQuantizedInt32( - input[i * elements_per_channel + j], scales[i]); - } - } -} - void SignedSymmetricPerChannelQuantize(const float* values, TfLiteIntArray* dims, int quantized_dimension, @@ -186,94 +64,17 @@ void SignedSymmetricPerChannelQuantize(const float* values, max = fmaxf(max, values[idx]); } scaling_factors[channel] = - fmaxf(fabs(min), fabs(max)) / kSymmetricInt8Scale; + fmaxf(fabs(min), fabs(max)) / std::numeric_limits::max(); for (int i = 0; i < per_channel_size; i++) { int idx = channel * channel_stride + i * stride; const int32_t quantized_value = static_cast(roundf(values[idx] / scaling_factors[channel])); // Clamp: just in case some odd numeric offset. - quantized_values[idx] = fminf( - kSymmetricInt8Scale, fmaxf(-kSymmetricInt8Scale, quantized_value)); + quantized_values[idx] = + fminf(std::numeric_limits::max(), + fmaxf(std::numeric_limits::min() + 1, quantized_value)); } } } -void SignedSymmetricQuantize(const float* values, TfLiteIntArray* dims, - int8_t* quantized_values, float* scaling_factor) { - int input_size = ElementCount(*dims); - - float min = 0; - float max = 0; - for (int i = 0; i < input_size; i++) { - min = fminf(min, values[i]); - max = fmaxf(max, values[i]); - } - *scaling_factor = fmaxf(fabs(min), fabs(max)) / kSymmetricInt8Scale; - for (int i = 0; i < input_size; i++) { - const int32_t quantized_value = - static_cast(roundf(values[i] / *scaling_factor)); - // Clamp: just in case some odd numeric offset. - quantized_values[i] = fminf(kSymmetricInt8Scale, - fmaxf(-kSymmetricInt8Scale, quantized_value)); - } -} - -void SignedSymmetricQuantize(const float* values, TfLiteIntArray* dims, - int16_t* quantized_values, float* scaling_factor) { - int input_size = ElementCount(*dims); - - float min = 0; - float max = 0; - for (int i = 0; i < input_size; i++) { - min = fminf(min, values[i]); - max = fmaxf(max, values[i]); - } - *scaling_factor = fmaxf(fabs(min), fabs(max)) / kSymmetricInt16Scale; - for (int i = 0; i < input_size; i++) { - const int32_t quantized_value = - static_cast(roundf(values[i] / *scaling_factor)); - // Clamp: just in case some odd numeric offset. - quantized_values[i] = fminf(kSymmetricInt16Scale, - fmaxf(-kSymmetricInt16Scale, quantized_value)); - } -} - -void SignedSymmetricQuantize(const float* values, TfLiteIntArray* dims, - int32_t* quantized_values, float* scaling_factor) { - int input_size = ElementCount(*dims); - - float min = 0; - float max = 0; - for (int i = 0; i < input_size; i++) { - min = fminf(min, values[i]); - max = fmaxf(max, values[i]); - } - - *scaling_factor = - fmaxf(fabs(min), fabs(max)) / static_cast(kSymmetricInt32Scale); - for (int i = 0; i < input_size; i++) { - const int32_t quantized_value = - static_cast(roundf(values[i] / *scaling_factor)); - // Clamp: just in case some odd numeric offset. - quantized_values[i] = fminf( - static_cast(kSymmetricInt32Scale), - fmaxf(static_cast(-kSymmetricInt32Scale), quantized_value)); - } -} - -void SymmetricQuantize(const float* values, TfLiteIntArray* dims, - uint8_t* quantized_values, float* scaling_factor) { - SignedSymmetricQuantize(values, dims, - reinterpret_cast(quantized_values), - scaling_factor); -} - -void SymmetricDequantize(const int8_t* values, const int size, - const float dequantization_scale, - float* dequantized_values) { - for (int i = 0; i < size; ++i) { - dequantized_values[i] = values[i] * dequantization_scale; - } -} - } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/micro_utils.h b/code/lib/tfmicro/tensorflow/lite/micro/micro_utils.h index 4f8689b9..b9a3121a 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/micro_utils.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/micro_utils.h @@ -16,7 +16,9 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_MICRO_UTILS_H_ #define TENSORFLOW_LITE_MICRO_MICRO_UTILS_H_ -#include +#include +#include +#include #include "tensorflow/lite/c/common.h" @@ -26,51 +28,68 @@ namespace tflite { int ElementCount(const TfLiteIntArray& dims); -uint8_t FloatToAsymmetricQuantizedUInt8(const float value, const float scale, - const int zero_point); +// Converts a float value into a quantized value. Note that large values (close +// to max int and min int) may see significant error due to a lack of floating +// point granularity for large values. +template +T FloatToQuantizedType(const float value, const float scale, int zero_point) { + int32_t result = round(value / scale) + zero_point; + result = + std::max(static_cast(std::numeric_limits::min()), result); + result = + std::min(static_cast(std::numeric_limits::max()), result); + return result; +} -uint8_t FloatToSymmetricQuantizedUInt8(const float value, const float scale); - -int8_t FloatToAsymmetricQuantizedInt8(const float value, const float scale, - const int zero_point); - -int16_t FloatToAsymmetricQuantizedInt16(const float value, const float scale, - const int zero_point); - -int8_t FloatToSymmetricQuantizedInt8(const float value, const float scale); - -// Converts a float value into a signed thirty-two-bit quantized value. Note -// that values close to max int and min int may see significant error due to -// a lack of floating point granularity for large values. -int32_t FloatToSymmetricQuantizedInt32(const float value, const float scale); +template +T FloatToSymmetricQuantizedType(const float value, const float scale) { + int32_t result = round(value / scale); + result = + std::max(static_cast(std::numeric_limits::min() + 1), result); + result = + std::min(static_cast(std::numeric_limits::max()), result); + return result; +} // Helper methods to quantize arrays of floats to the desired format. // // There are several key flavors of quantization in TfLite: // asymmetric symmetric per channel -// int8 | X | X | X | -// uint8 | X | X | | -// int16 | X | | | -// int32 | | X | X | +// int8_t | X | X | X | +// uint8_t | X | X | | +// int16_t | X | | | +// int32_t | | X | X | // // The per-op quantization spec can be found here: // https://www.tensorflow.org/lite/performance/quantization_spec +template +void Quantize(const float* input, T* output, int num_elements, float scale, + int zero_point) { + for (int i = 0; i < num_elements; i++) { + output[i] = FloatToQuantizedType(input[i], scale, zero_point); + } +} -void AsymmetricQuantize(const float* input, int8_t* output, int num_elements, - float scale, int zero_point = 0); +template +void SymmetricQuantize(const float* input, T* output, int num_elements, + float scale) { + for (int i = 0; i < num_elements; i++) { + output[i] = FloatToSymmetricQuantizedType(input[i], scale); + } +} -void AsymmetricQuantize(const float* input, uint8_t* output, int num_elements, - float scale, int zero_point = 128); - -void AsymmetricQuantize(const float* input, int16_t* output, int num_elements, - float scale, int zero_point = 0); - -void SymmetricQuantize(const float* input, int32_t* output, int num_elements, - float scale); - -void SymmetricPerChannelQuantize(const float* input, int32_t* output, +template +void SymmetricPerChannelQuantize(const float* input, T* output, int num_elements, int num_channels, - float* scales); + float* scales) { + int elements_per_channel = num_elements / num_channels; + for (int i = 0; i < num_channels; i++) { + for (int j = 0; j < elements_per_channel; j++) { + output[i * elements_per_channel + j] = FloatToSymmetricQuantizedType( + input[i * elements_per_channel + j], scales[i]); + } + } +} void SignedSymmetricPerChannelQuantize(const float* values, TfLiteIntArray* dims, @@ -78,21 +97,37 @@ void SignedSymmetricPerChannelQuantize(const float* values, int8_t* quantized_values, float* scaling_factor); -void SignedSymmetricQuantize(const float* values, TfLiteIntArray* dims, - int8_t* quantized_values, float* scaling_factor); +// Quantizes inputs based on the values provided, choosing the smallest range +// which includes all input values. +template +void SymmetricQuantizeCalculateScales(const float* values, TfLiteIntArray* dims, + T* output, float* scale) { + int input_size = ElementCount(*dims); -void SignedSymmetricQuantize(const float* values, TfLiteIntArray* dims, - int16_t* quantized_values, float* scaling_factor); + float min = 0; + float max = 0; + for (int i = 0; i < input_size; i++) { + min = fminf(min, values[i]); + max = fmaxf(max, values[i]); + } + *scale = fmaxf(std::abs(min), std::abs(max)) / std::numeric_limits::max(); + for (int i = 0; i < input_size; i++) { + const int32_t quantized_value = + static_cast(roundf(values[i] / *scale)); + // Clamp: just in case some odd numeric offset. + quantized_value = fminf(std::numeric_limits::max(), quantized_value); + quantized_value = fmaxf(std::numeric_limits::min() + 1, quantized_value); + output[i] = quantized_value; + } +} -void SignedSymmetricQuantize(const float* values, TfLiteIntArray* dims, - int32_t* quantized_values, float* scaling_factor); - -void SymmetricQuantize(const float* values, TfLiteIntArray* dims, - uint8_t* quantized_values, float* scaling_factor); - -void SymmetricDequantize(const int8_t* values, const int size, - const float dequantization_scale, - float* dequantized_values); +template +void Dequantize(const T* values, const int size, const float scale, + int zero_point, float* dequantized_values) { + for (int i = 0; i < size; ++i) { + dequantized_values[i] = (values[i] - zero_point) * scale; + } +} } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_allocator.cc b/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_allocator.cc new file mode 100644 index 00000000..6bb52974 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_allocator.cc @@ -0,0 +1,244 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/recording_micro_allocator.h" + +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/compatibility.h" +#include "tensorflow/lite/micro/micro_allocator.h" +#include "tensorflow/lite/micro/recording_simple_memory_allocator.h" + +namespace tflite { + +RecordingMicroAllocator::RecordingMicroAllocator( + RecordingSimpleMemoryAllocator* recording_memory_allocator, + ErrorReporter* error_reporter) + : MicroAllocator(recording_memory_allocator, error_reporter), + recording_memory_allocator_(recording_memory_allocator) {} + +RecordingMicroAllocator* RecordingMicroAllocator::Create( + uint8_t* tensor_arena, size_t arena_size, ErrorReporter* error_reporter) { + TFLITE_DCHECK(error_reporter != nullptr); + + RecordingSimpleMemoryAllocator* simple_memory_allocator = + RecordingSimpleMemoryAllocator::Create(error_reporter, tensor_arena, + arena_size); + TFLITE_DCHECK(simple_memory_allocator != nullptr); + + uint8_t* allocator_buffer = simple_memory_allocator->AllocateFromTail( + sizeof(RecordingMicroAllocator), alignof(RecordingMicroAllocator)); + RecordingMicroAllocator* allocator = new (allocator_buffer) + RecordingMicroAllocator(simple_memory_allocator, error_reporter); + return allocator; +} + +RecordedAllocation RecordingMicroAllocator::GetRecordedAllocation( + RecordedAllocationType allocation_type) const { + switch (allocation_type) { + case RecordedAllocationType::kTfLiteEvalTensorData: + return recorded_tflite_eval_tensor_data_; + case RecordedAllocationType::kPersistentTfLiteTensorData: + return recorded_persistent_tflite_tensor_data_; + case RecordedAllocationType::kPersistentTfLiteTensorQuantizationData: + return recorded_persistent_tflite_tensor_quantization_data_; + case RecordedAllocationType::kPersistentBufferData: + return recorded_persistent_buffer_data_; + case RecordedAllocationType::kTfLiteTensorVariableBufferData: + return recorded_tflite_tensor_variable_buffer_data_; + case RecordedAllocationType::kNodeAndRegistrationArray: + return recorded_node_and_registration_array_data_; + case RecordedAllocationType::kOpData: + return recorded_op_data_; + } + TF_LITE_REPORT_ERROR(error_reporter(), "Invalid allocation type supplied: %d", + allocation_type); + return RecordedAllocation(); +} + +const RecordingSimpleMemoryAllocator* +RecordingMicroAllocator::GetSimpleMemoryAllocator() const { + return recording_memory_allocator_; +} + +void RecordingMicroAllocator::PrintAllocations() const { + TF_LITE_REPORT_ERROR( + error_reporter(), + "[RecordingMicroAllocator] Arena allocation total %d bytes", + recording_memory_allocator_->GetUsedBytes()); + TF_LITE_REPORT_ERROR( + error_reporter(), + "[RecordingMicroAllocator] Arena allocation head %d bytes", + recording_memory_allocator_->GetHeadUsedBytes()); + TF_LITE_REPORT_ERROR( + error_reporter(), + "[RecordingMicroAllocator] Arena allocation tail %d bytes", + recording_memory_allocator_->GetTailUsedBytes()); + PrintRecordedAllocation(RecordedAllocationType::kTfLiteEvalTensorData, + "TfLiteEvalTensor data", "allocations"); + PrintRecordedAllocation(RecordedAllocationType::kPersistentTfLiteTensorData, + "Persistent TfLiteTensor data", "tensors"); + PrintRecordedAllocation( + RecordedAllocationType::kPersistentTfLiteTensorQuantizationData, + "Persistent TfLiteTensor quantization data", "allocations"); + PrintRecordedAllocation(RecordedAllocationType::kPersistentBufferData, + "Persistent buffer data", "allocations"); + PrintRecordedAllocation( + RecordedAllocationType::kTfLiteTensorVariableBufferData, + "TfLiteTensor variable buffer data", "allocations"); + PrintRecordedAllocation(RecordedAllocationType::kNodeAndRegistrationArray, + "NodeAndRegistration struct", + "NodeAndRegistration structs"); + PrintRecordedAllocation(RecordedAllocationType::kOpData, + "Operator runtime data", "OpData structs"); +} + +void* RecordingMicroAllocator::AllocatePersistentBuffer(size_t bytes) { + RecordedAllocation allocations = SnapshotAllocationUsage(); + void* buffer = MicroAllocator::AllocatePersistentBuffer(bytes); + RecordAllocationUsage(allocations, recorded_persistent_buffer_data_); + + return buffer; +} + +void RecordingMicroAllocator::PrintRecordedAllocation( + RecordedAllocationType allocation_type, const char* allocation_name, + const char* allocation_description) const { +#ifndef TF_LITE_STRIP_ERROR_STRINGS + RecordedAllocation allocation = GetRecordedAllocation(allocation_type); + if (allocation.used_bytes > 0 || allocation.requested_bytes > 0) { + TF_LITE_REPORT_ERROR( + error_reporter(), + "[RecordingMicroAllocator] '%s' used %d bytes with alignment overhead " + "(requested %d bytes for %d %s)", + allocation_name, allocation.used_bytes, allocation.requested_bytes, + allocation.count, allocation_description); + } +#endif +} + +TfLiteStatus RecordingMicroAllocator::AllocateNodeAndRegistrations( + const Model* model, NodeAndRegistration** node_and_registrations) { + RecordedAllocation allocations = SnapshotAllocationUsage(); + + TfLiteStatus status = MicroAllocator::AllocateNodeAndRegistrations( + model, node_and_registrations); + + RecordAllocationUsage(allocations, + recorded_node_and_registration_array_data_); + // The allocation count in SimpleMemoryAllocator will only be 1. To provide + // better logging, decrement by 1 and add in the actual number of operators + // used in the graph: + // The allocation for this recording will always be 1. This is because the + // parent class mallocs one large allocation for the number of nodes in the + // graph (e.g. sizeof(NodeAndRegistration) * num_nodes). + // To prevent extra overhead and potential for fragmentation, manually adjust + // the accounting by decrementing by 1 and adding the actual number of nodes + // used in the graph: + recorded_node_and_registration_array_data_.count += + GetSubGraphFromModel(model)->operators()->size() - 1; + return status; +} + +TfLiteStatus +RecordingMicroAllocator::PrepareNodeAndRegistrationDataFromFlatbuffer( + const Model* model, const MicroOpResolver& op_resolver, + NodeAndRegistration* node_and_registrations) { + RecordedAllocation allocations = SnapshotAllocationUsage(); + + TfLiteStatus status = + MicroAllocator::PrepareNodeAndRegistrationDataFromFlatbuffer( + model, op_resolver, node_and_registrations); + + RecordAllocationUsage(allocations, recorded_op_data_); + return status; +} + +TfLiteStatus RecordingMicroAllocator::AllocateTfLiteEvalTensors( + const Model* model, TfLiteEvalTensor** eval_tensors) { + RecordedAllocation allocations = SnapshotAllocationUsage(); + + TfLiteStatus status = + MicroAllocator::AllocateTfLiteEvalTensors(model, eval_tensors); + + RecordAllocationUsage(allocations, recorded_tflite_eval_tensor_data_); + // The allocation for this recording will always be 1. This is because the + // parent class mallocs one large allocation for the number of tensors in the + // graph (e.g. sizeof(TfLiteEvalTensor) * num_tensors). + // To prevent extra overhead and potential for fragmentation, manually adjust + // the accounting by decrementing by 1 and adding the actual number of tensors + // used in the graph: + recorded_tflite_eval_tensor_data_.count += + GetSubGraphFromModel(model)->tensors()->size() - 1; + return status; +} + +TfLiteStatus RecordingMicroAllocator::AllocateVariables( + const SubGraph* subgraph, TfLiteEvalTensor* eval_tensors) { + RecordedAllocation allocations = SnapshotAllocationUsage(); + + TfLiteStatus status = + MicroAllocator::AllocateVariables(subgraph, eval_tensors); + + RecordAllocationUsage(allocations, + recorded_tflite_tensor_variable_buffer_data_); + return status; +} + +TfLiteTensor* RecordingMicroAllocator::AllocatePersistentTfLiteTensorInternal( + const Model* model, TfLiteEvalTensor* eval_tensors, int tensor_index) { + RecordedAllocation allocations = SnapshotAllocationUsage(); + + TfLiteTensor* result = MicroAllocator::AllocatePersistentTfLiteTensorInternal( + model, eval_tensors, tensor_index); + + RecordAllocationUsage(allocations, recorded_persistent_tflite_tensor_data_); + return result; +} + +TfLiteStatus RecordingMicroAllocator::PopulateTfLiteTensorFromFlatbuffer( + const Model* model, const SubGraph* subgraph, TfLiteTensor* tensor, + int tensor_index, bool allocate_temp) { + RecordedAllocation allocations = SnapshotAllocationUsage(); + + TfLiteStatus status = MicroAllocator::PopulateTfLiteTensorFromFlatbuffer( + model, subgraph, tensor, tensor_index, allocate_temp); + + RecordAllocationUsage(allocations, + recorded_persistent_tflite_tensor_quantization_data_); + return status; +} + +RecordedAllocation RecordingMicroAllocator::SnapshotAllocationUsage() const { + return {/*requested_bytes=*/recording_memory_allocator_->GetRequestedBytes(), + /*used_bytes=*/recording_memory_allocator_->GetUsedBytes(), + /*count=*/recording_memory_allocator_->GetAllocatedCount()}; +} + +void RecordingMicroAllocator::RecordAllocationUsage( + const RecordedAllocation& snapshotted_allocation, + RecordedAllocation& recorded_allocation) { + recorded_allocation.requested_bytes += + recording_memory_allocator_->GetRequestedBytes() - + snapshotted_allocation.requested_bytes; + recorded_allocation.used_bytes += + recording_memory_allocator_->GetUsedBytes() - + snapshotted_allocation.used_bytes; + recorded_allocation.count += + recording_memory_allocator_->GetAllocatedCount() - + snapshotted_allocation.count; +} + +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_allocator.h b/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_allocator.h new file mode 100644 index 00000000..47246e17 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_allocator.h @@ -0,0 +1,125 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_RECORDING_MICRO_ALLOCATOR_H_ +#define TENSORFLOW_LITE_MICRO_RECORDING_MICRO_ALLOCATOR_H_ + +#include "tensorflow/lite/micro/compatibility.h" +#include "tensorflow/lite/micro/micro_allocator.h" +#include "tensorflow/lite/micro/recording_simple_memory_allocator.h" + +namespace tflite { + +// List of buckets currently recorded by this class. Each type keeps a list of +// allocated information during model initialization. +// TODO(b/169834511): Add tracking for scratch buffer allocations. +enum class RecordedAllocationType { + kTfLiteEvalTensorData, + kPersistentTfLiteTensorData, + kPersistentTfLiteTensorQuantizationData, + kPersistentBufferData, + kTfLiteTensorVariableBufferData, + kNodeAndRegistrationArray, + kOpData, +}; + +// Container for holding information about allocation recordings by a given +// type. Each recording contains the number of bytes requested, the actual bytes +// allocated (can defer from requested by alignment), and the number of items +// allocated. +struct RecordedAllocation { + size_t requested_bytes; + size_t used_bytes; + size_t count; +}; + +// Utility subclass of MicroAllocator that records all allocations +// inside the arena. A summary of allocations can be logged through the +// ErrorReporter by invoking LogAllocations(). This special allocator requires +// an instance of RecordingSimpleMemoryAllocator to capture allocations in the +// head and tail. Arena allocation recording can be retrieved by type through +// the GetRecordedAllocation() function. This class should only be used for +// auditing memory usage or integration testing. +class RecordingMicroAllocator : public MicroAllocator { + public: + static RecordingMicroAllocator* Create(uint8_t* tensor_arena, + size_t arena_size, + ErrorReporter* error_reporter); + + // Returns the recorded allocations information for a given allocation type. + RecordedAllocation GetRecordedAllocation( + RecordedAllocationType allocation_type) const; + + const RecordingSimpleMemoryAllocator* GetSimpleMemoryAllocator() const; + + // Logs out through the ErrorReporter all allocation recordings by type + // defined in RecordedAllocationType. + void PrintAllocations() const; + + void* AllocatePersistentBuffer(size_t bytes) override; + + protected: + TfLiteStatus AllocateNodeAndRegistrations( + const Model* model, + NodeAndRegistration** node_and_registrations) override; + TfLiteStatus PrepareNodeAndRegistrationDataFromFlatbuffer( + const Model* model, const MicroOpResolver& op_resolver, + NodeAndRegistration* node_and_registrations) override; + TfLiteStatus AllocateTfLiteEvalTensors( + const Model* model, TfLiteEvalTensor** eval_tensors) override; + TfLiteStatus AllocateVariables(const SubGraph* subgraph, + TfLiteEvalTensor* eval_tensors) override; + // TODO(b/162311891): Once all kernels have been updated to the new API drop + // this method. It is only used to record TfLiteTensor persistent allocations. + TfLiteTensor* AllocatePersistentTfLiteTensorInternal( + const Model* model, TfLiteEvalTensor* eval_tensors, + int tensor_index) override; + // TODO(b/162311891): Once all kernels have been updated to the new API drop + // this function since all allocations for quantized data will take place in + // the temp section. + TfLiteStatus PopulateTfLiteTensorFromFlatbuffer(const Model* model, + const SubGraph* subgraph, + TfLiteTensor* tensor, + int tensor_index, + bool allocate_temp) override; + + private: + RecordingMicroAllocator(RecordingSimpleMemoryAllocator* memory_allocator, + ErrorReporter* error_reporter); + + void PrintRecordedAllocation(RecordedAllocationType allocation_type, + const char* allocation_name, + const char* allocation_description) const; + + RecordedAllocation SnapshotAllocationUsage() const; + void RecordAllocationUsage(const RecordedAllocation& snapshotted_allocation, + RecordedAllocation& recorded_allocation); + + const RecordingSimpleMemoryAllocator* recording_memory_allocator_; + + RecordedAllocation recorded_tflite_eval_tensor_data_ = {}; + RecordedAllocation recorded_persistent_tflite_tensor_data_ = {}; + RecordedAllocation recorded_persistent_tflite_tensor_quantization_data_ = {}; + RecordedAllocation recorded_persistent_buffer_data_ = {}; + RecordedAllocation recorded_tflite_tensor_variable_buffer_data_ = {}; + RecordedAllocation recorded_node_and_registration_array_data_ = {}; + RecordedAllocation recorded_op_data_ = {}; + + TF_LITE_REMOVE_VIRTUAL_DELETE +}; + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_RECORDING_MICRO_ALLOCATOR_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_interpreter.h b/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_interpreter.h new file mode 100644 index 00000000..0a579b0b --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/recording_micro_interpreter.h @@ -0,0 +1,65 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_RECORDING_MICRO_INTERPRETER_H_ +#define TENSORFLOW_LITE_MICRO_RECORDING_MICRO_INTERPRETER_H_ + +#include "tensorflow/lite/micro/micro_interpreter.h" +#include "tensorflow/lite/micro/recording_micro_allocator.h" + +namespace tflite { + +// Utility subclass that enables internal recordings of the MicroInterpreter. +// This class should be used to audit and analyze memory arena usage for a given +// model and interpreter. +// +// After construction and the first Invoke() or AllocateTensors() call - the +// memory usage is recorded and available through the GetMicroAllocator() +// function. See RecordingMicroAlloctor for more details on what is currently +// recorded from arena allocations. +// +// It is recommended for users to increase the tensor arena size by at least 1kb +// to ensure enough additional memory is available for internal recordings. +class RecordingMicroInterpreter : public MicroInterpreter { + public: + RecordingMicroInterpreter(const Model* model, + const MicroOpResolver& op_resolver, + uint8_t* tensor_arena, size_t tensor_arena_size, + ErrorReporter* error_reporter) + : MicroInterpreter(model, op_resolver, + RecordingMicroAllocator::Create( + tensor_arena, tensor_arena_size, error_reporter), + error_reporter), + recording_micro_allocator_( + static_cast(allocator())) {} + + RecordingMicroInterpreter(const Model* model, + const MicroOpResolver& op_resolver, + RecordingMicroAllocator* allocator, + ErrorReporter* error_reporter) + : MicroInterpreter(model, op_resolver, allocator, error_reporter), + recording_micro_allocator_(*allocator) {} + + const RecordingMicroAllocator& GetMicroAllocator() const { + return recording_micro_allocator_; + } + + private: + const RecordingMicroAllocator& recording_micro_allocator_; +}; + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_RECORDING_MICRO_INTERPRETER_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/recording_simple_memory_allocator.cc b/code/lib/tfmicro/tensorflow/lite/micro/recording_simple_memory_allocator.cc new file mode 100644 index 00000000..ef30aca4 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/recording_simple_memory_allocator.cc @@ -0,0 +1,84 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/recording_simple_memory_allocator.h" + +#include + +#include "tensorflow/lite/kernels/internal/compatibility.h" + +namespace tflite { + +RecordingSimpleMemoryAllocator::RecordingSimpleMemoryAllocator( + ErrorReporter* error_reporter, uint8_t* buffer_head, size_t buffer_size) + : SimpleMemoryAllocator(error_reporter, buffer_head, buffer_size), + requested_head_bytes_(0), + requested_tail_bytes_(0), + used_bytes_(0), + alloc_count_(0) {} + +RecordingSimpleMemoryAllocator::~RecordingSimpleMemoryAllocator() {} + +RecordingSimpleMemoryAllocator* RecordingSimpleMemoryAllocator::Create( + ErrorReporter* error_reporter, uint8_t* buffer_head, size_t buffer_size) { + TFLITE_DCHECK(error_reporter != nullptr); + TFLITE_DCHECK(buffer_head != nullptr); + RecordingSimpleMemoryAllocator tmp = + RecordingSimpleMemoryAllocator(error_reporter, buffer_head, buffer_size); + + uint8_t* allocator_buffer = + tmp.AllocateFromTail(sizeof(RecordingSimpleMemoryAllocator), + alignof(RecordingSimpleMemoryAllocator)); + // Use the default copy constructor to populate internal states. + return new (allocator_buffer) RecordingSimpleMemoryAllocator(tmp); +} + +size_t RecordingSimpleMemoryAllocator::GetRequestedBytes() const { + return requested_head_bytes_ + requested_tail_bytes_; +} + +size_t RecordingSimpleMemoryAllocator::GetUsedBytes() const { + return used_bytes_; +} + +size_t RecordingSimpleMemoryAllocator::GetAllocatedCount() const { + return alloc_count_; +} + +TfLiteStatus RecordingSimpleMemoryAllocator::SetHeadBufferSize( + size_t size, size_t alignment) { + const uint8_t* previous_head = head(); + TfLiteStatus status = + SimpleMemoryAllocator::SetHeadBufferSize(size, alignment); + if (status == kTfLiteOk) { + used_bytes_ += head() - previous_head; + requested_head_bytes_ = size; + } + return status; +} + +uint8_t* RecordingSimpleMemoryAllocator::AllocateFromTail(size_t size, + size_t alignment) { + const uint8_t* previous_tail = tail(); + uint8_t* result = SimpleMemoryAllocator::AllocateFromTail(size, alignment); + if (result != nullptr) { + used_bytes_ += previous_tail - tail(); + requested_tail_bytes_ += size; + alloc_count_++; + } + return result; +} + +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/recording_simple_memory_allocator.h b/code/lib/tfmicro/tensorflow/lite/micro/recording_simple_memory_allocator.h new file mode 100644 index 00000000..3526716e --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/micro/recording_simple_memory_allocator.h @@ -0,0 +1,64 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_RECORDING_SIMPLE_MEMORY_ALLOCATOR_H_ +#define TENSORFLOW_LITE_MICRO_RECORDING_SIMPLE_MEMORY_ALLOCATOR_H_ + +#include "tensorflow/lite/micro/compatibility.h" +#include "tensorflow/lite/micro/simple_memory_allocator.h" + +namespace tflite { + +// Utility class used to log allocations of a SimpleMemoryAllocator. Should only +// be used in debug/evaluation settings or unit tests to evaluate allocation +// usage. +class RecordingSimpleMemoryAllocator : public SimpleMemoryAllocator { + public: + RecordingSimpleMemoryAllocator(ErrorReporter* error_reporter, + uint8_t* buffer_head, size_t buffer_size); + // TODO(b/157615197): Cleanup constructors/destructor and use factory + // functions. + ~RecordingSimpleMemoryAllocator() override; + + static RecordingSimpleMemoryAllocator* Create(ErrorReporter* error_reporter, + uint8_t* buffer_head, + size_t buffer_size); + + // Returns the number of bytes requested from the head or tail. + size_t GetRequestedBytes() const; + + // Returns the number of bytes actually allocated from the head or tail. This + // value will be >= to the number of requested bytes due to padding and + // alignment. + size_t GetUsedBytes() const; + + // Returns the number of alloc calls from the head or tail. + size_t GetAllocatedCount() const; + + TfLiteStatus SetHeadBufferSize(size_t size, size_t alignment) override; + uint8_t* AllocateFromTail(size_t size, size_t alignment) override; + + private: + size_t requested_head_bytes_; + size_t requested_tail_bytes_; + size_t used_bytes_; + size_t alloc_count_; + + TF_LITE_REMOVE_VIRTUAL_DELETE +}; + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_RECORDING_SIMPLE_MEMORY_ALLOCATOR_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/micro/simple_memory_allocator.cc b/code/lib/tfmicro/tensorflow/lite/micro/simple_memory_allocator.cc index be7c4695..08b6789e 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/simple_memory_allocator.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/simple_memory_allocator.cc @@ -1,4 +1,4 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,50 +17,133 @@ limitations under the License. #include #include +#include +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/micro/memory_helpers.h" namespace tflite { -SimpleMemoryAllocator* CreateInPlaceSimpleMemoryAllocator( - ErrorReporter* error_reporter, uint8_t* buffer, size_t buffer_size) { +SimpleMemoryAllocator::SimpleMemoryAllocator(ErrorReporter* error_reporter, + uint8_t* buffer_head, + uint8_t* buffer_tail) + : error_reporter_(error_reporter), + buffer_head_(buffer_head), + buffer_tail_(buffer_tail), + head_(buffer_head), + tail_(buffer_tail), + temp_(buffer_head_) {} + +SimpleMemoryAllocator::SimpleMemoryAllocator(ErrorReporter* error_reporter, + uint8_t* buffer, + size_t buffer_size) + : SimpleMemoryAllocator(error_reporter, buffer, buffer + buffer_size) {} + +/* static */ +SimpleMemoryAllocator* SimpleMemoryAllocator::Create( + ErrorReporter* error_reporter, uint8_t* buffer_head, size_t buffer_size) { + TFLITE_DCHECK(error_reporter != nullptr); + TFLITE_DCHECK(buffer_head != nullptr); SimpleMemoryAllocator tmp = - SimpleMemoryAllocator(error_reporter, buffer, buffer_size); - SimpleMemoryAllocator* in_place_allocator = - reinterpret_cast(tmp.AllocateFromTail( - sizeof(SimpleMemoryAllocator), alignof(SimpleMemoryAllocator))); - *in_place_allocator = tmp; - return in_place_allocator; + SimpleMemoryAllocator(error_reporter, buffer_head, buffer_size); + + // Allocate enough bytes from the buffer to create a SimpleMemoryAllocator. + // The new instance will use the current adjusted tail buffer from the tmp + // allocator instance. + uint8_t* allocator_buffer = tmp.AllocateFromTail( + sizeof(SimpleMemoryAllocator), alignof(SimpleMemoryAllocator)); + // Use the default copy constructor to populate internal states. + return new (allocator_buffer) SimpleMemoryAllocator(tmp); } -uint8_t* SimpleMemoryAllocator::AllocateFromHead(size_t size, - size_t alignment) { - uint8_t* const aligned_result = AlignPointerUp(head_, alignment); +SimpleMemoryAllocator::~SimpleMemoryAllocator() {} + +TfLiteStatus SimpleMemoryAllocator::SetHeadBufferSize(size_t size, + size_t alignment) { + if (head_ != temp_) { + TF_LITE_REPORT_ERROR( + error_reporter_, + "Internal error: SetHeadBufferSize() needs to be called " + "after ResetTempAllocations()."); + return kTfLiteError; + } + + uint8_t* const aligned_result = AlignPointerUp(buffer_head_, alignment); const size_t available_memory = tail_ - aligned_result; if (available_memory < size) { TF_LITE_REPORT_ERROR( error_reporter_, - "Failed to allocate memory. Requested: %u, available %u, missing: %u", + "Failed to set head size. Requested: %u, available %u, missing: %u", size, available_memory, size - available_memory); - return nullptr; + return kTfLiteError; } head_ = aligned_result + size; - return aligned_result; + temp_ = head_; + + return kTfLiteOk; } uint8_t* SimpleMemoryAllocator::AllocateFromTail(size_t size, size_t alignment) { uint8_t* const aligned_result = AlignPointerDown(tail_ - size, alignment); if (aligned_result < head_) { +#ifndef TF_LITE_STRIP_ERROR_STRINGS const size_t missing_memory = head_ - aligned_result; - TF_LITE_REPORT_ERROR( - error_reporter_, - "Failed to allocate memory. Requested: %u, available %u, missing: %u", - size, size - missing_memory, missing_memory); + TF_LITE_REPORT_ERROR(error_reporter_, + "Failed to allocate tail memory. Requested: %u, " + "available %u, missing: %u", + size, size - missing_memory, missing_memory); +#endif return nullptr; } tail_ = aligned_result; return aligned_result; } +uint8_t* SimpleMemoryAllocator::AllocateTemp(size_t size, size_t alignment) { + uint8_t* const aligned_result = AlignPointerUp(temp_, alignment); + const size_t available_memory = tail_ - aligned_result; + if (available_memory < size) { + TF_LITE_REPORT_ERROR(error_reporter_, + "Failed to allocate temp memory. Requested: %u, " + "available %u, missing: %u", + size, available_memory, size - available_memory); + return nullptr; + } + temp_ = aligned_result + size; + return aligned_result; +} + +void SimpleMemoryAllocator::ResetTempAllocations() { temp_ = head_; } + +uint8_t* SimpleMemoryAllocator::GetHeadBuffer() const { return buffer_head_; } + +size_t SimpleMemoryAllocator::GetHeadUsedBytes() const { + return head_ - buffer_head_; +} + +size_t SimpleMemoryAllocator::GetTailUsedBytes() const { + return buffer_tail_ - tail_; +} + +size_t SimpleMemoryAllocator::GetAvailableMemory(size_t alignment) const { + uint8_t* const aligned_temp = AlignPointerUp(temp_, alignment); + uint8_t* const aligned_tail = AlignPointerDown(tail_, alignment); + return aligned_tail - aligned_temp; +} + +size_t SimpleMemoryAllocator::GetUsedBytes() const { + return GetBufferSize() - (tail_ - temp_); +} + +size_t SimpleMemoryAllocator::GetBufferSize() const { + return buffer_tail_ - buffer_head_; +} + +uint8_t* SimpleMemoryAllocator::head() const { return head_; } + +uint8_t* SimpleMemoryAllocator::tail() const { return tail_; } + } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/simple_memory_allocator.h b/code/lib/tfmicro/tensorflow/lite/micro/simple_memory_allocator.h index cf181860..35adaf1a 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/simple_memory_allocator.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/simple_memory_allocator.h @@ -1,4 +1,4 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,10 +16,12 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_SIMPLE_MEMORY_ALLOCATOR_H_ #define TENSORFLOW_LITE_MICRO_SIMPLE_MEMORY_ALLOCATOR_H_ +#include #include #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/micro/compatibility.h" namespace tflite { @@ -28,43 +30,82 @@ namespace tflite { // This makes it pretty wasteful, so we should use a more intelligent method. class SimpleMemoryAllocator { public: + // TODO(b/157615197): Cleanup constructors/destructor and use factory + // functions. SimpleMemoryAllocator(ErrorReporter* error_reporter, uint8_t* buffer_head, - uint8_t* buffer_tail) - : error_reporter_(error_reporter), - buffer_head_(buffer_head), - buffer_tail_(buffer_tail), - head_(buffer_head), - tail_(buffer_tail) {} + uint8_t* buffer_tail); SimpleMemoryAllocator(ErrorReporter* error_reporter, uint8_t* buffer, - size_t buffer_size) - : SimpleMemoryAllocator(error_reporter, buffer, buffer + buffer_size) {} + size_t buffer_size); + virtual ~SimpleMemoryAllocator(); + + // Creates a new SimpleMemoryAllocator from a given buffer head and size. + static SimpleMemoryAllocator* Create(ErrorReporter* error_reporter, + uint8_t* buffer_head, + size_t buffer_size); + + // Adjusts the head (lowest address and moving upwards) memory allocation to a + // given size. Calls to this method will also invalidate all temporary + // allocation values (it sets the location of temp space at the end of the + // head section). This call will fail if a chain of allocations through + // AllocateTemp() have not been cleaned up with a call to + // ResetTempAllocations(). + virtual TfLiteStatus SetHeadBufferSize(size_t size, size_t alignment); - // Allocates memory starting at the head of the arena (lowest address and - // moving upwards). - uint8_t* AllocateFromHead(size_t size, size_t alignment); // Allocates memory starting at the tail of the arena (highest address and // moving downwards). - uint8_t* AllocateFromTail(size_t size, size_t alignment); + virtual uint8_t* AllocateFromTail(size_t size, size_t alignment); - uint8_t* GetHead() const { return head_; } - uint8_t* GetTail() const { return tail_; } - size_t GetAvailableMemory() const { return tail_ - head_; } - size_t GetUsedBytes() const { return GetBufferSize() - GetAvailableMemory(); } + // Allocates a temporary buffer from the head of the arena (lowest address and + // moving upwards) but does not update the actual head allocation size or + // position. The returned buffer is guaranteed until either + // ResetTempAllocations() is called or another call to AllocateFromHead(). + // Repeat calls to this function will create a chain of temp allocations. All + // calls to AllocateTemp() must end with a call to ResetTempAllocations(). If + // AllocateFromHead() is called before a call to ResetTempAllocations(), it + // will fail with an error message. + virtual uint8_t* AllocateTemp(size_t size, size_t alignment); + + // Resets a chain of temporary allocations back to the current head of the + // arena (lowest address). + virtual void ResetTempAllocations(); + + // Returns a pointer to the buffer currently assigned to the head section. + // This buffer is set by calling SetHeadSize(). + uint8_t* GetHeadBuffer() const; + + // Returns the size of the head section in bytes. + size_t GetHeadUsedBytes() const; + + // Returns the size of all allocations in the tail section in bytes. + size_t GetTailUsedBytes() const; + + // Returns the number of bytes available with a given alignment. This number + // takes in account any temporary allocations. + size_t GetAvailableMemory(size_t alignment) const; + + // Returns the number of used bytes in the allocator. This number takes in + // account any temporary allocations. + size_t GetUsedBytes() const; + + protected: + // Returns a pointer to the current end of the head buffer. + uint8_t* head() const; + + // Returns a pointer to the current end of the tail buffer. + uint8_t* tail() const; private: - size_t GetBufferSize() const { return buffer_tail_ - buffer_head_; } + size_t GetBufferSize() const; ErrorReporter* error_reporter_; uint8_t* buffer_head_; uint8_t* buffer_tail_; uint8_t* head_; uint8_t* tail_; -}; + uint8_t* temp_; -// Allocate a SimpleMemoryAllocator from the buffer and then return the pointer -// to this allocator. -SimpleMemoryAllocator* CreateInPlaceSimpleMemoryAllocator( - ErrorReporter* error_reporter, uint8_t* buffer, size_t buffer_size); + TF_LITE_REMOVE_VIRTUAL_DELETE +}; } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/test_helpers.cc b/code/lib/tfmicro/tensorflow/lite/micro/test_helpers.cc index 77a1cc82..897f3110 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/test_helpers.cc +++ b/code/lib/tfmicro/tensorflow/lite/micro/test_helpers.cc @@ -15,14 +15,24 @@ limitations under the License. #include "tensorflow/lite/micro/test_helpers.h" +#include +#include +#include #include +#include +#include "flatbuffers/flatbuffers.h" // from @flatbuffers #include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/core/api/tensor_utils.h" +#include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/all_ops_resolver.h" #include "tensorflow/lite/micro/micro_utils.h" #include "tensorflow/lite/schema/schema_generated.h" +// TODO(b/170464050): Use TFLM test only version of schema_utils. + namespace tflite { namespace testing { namespace { @@ -48,7 +58,7 @@ class StackAllocator : public flatbuffers::Allocator { return *inst; } - static constexpr size_t kStackAllocatorSize = 4096; + static constexpr size_t kStackAllocatorSize = 8192; private: uint8_t data_backing_[kStackAllocatorSize]; @@ -76,8 +86,7 @@ class ModelBuilder { : builder_(builder) {} // Registers an operator that will be used in the model. - Operator RegisterOp(BuiltinOperator op, const char* custom_code, - int32_t version); + Operator RegisterOp(BuiltinOperator op, const char* custom_code); // Adds a tensor to the model. Tensor AddTensor(TensorType type, std::initializer_list shape) { @@ -94,10 +103,16 @@ class ModelBuilder { Node AddNode(Operator op, std::initializer_list inputs, std::initializer_list outputs); + void AddMetadata(const char* description_string, + const int32_t* metadata_buffer_data, size_t num_elements); + // Constructs the flatbuffer model using `builder_` and return a pointer to // it. The returned model has the same lifetime as `builder_`. + // Note the default value of 0 for num_subgraph_inputs means all tensor inputs + // are in subgraph input list. const Model* BuildModel(std::initializer_list inputs, - std::initializer_list outputs); + std::initializer_list outputs, + size_t num_subgraph_inputs = 0); private: // Adds a tensor to the model. @@ -116,15 +131,24 @@ class ModelBuilder { static constexpr int kMaxTensors = 50; flatbuffers::Offset tensors_[kMaxTensors]; + + static constexpr int kMaxMetadataBuffers = 10; + + static constexpr int kMaxMetadatas = 10; + flatbuffers::Offset metadata_[kMaxMetadatas]; + + flatbuffers::Offset metadata_buffers_[kMaxMetadataBuffers]; + + int nbr_of_metadata_buffers_ = 0; + int next_tensor_id_ = 0; }; ModelBuilder::Operator ModelBuilder::RegisterOp(BuiltinOperator op, - const char* custom_code, - int32_t version) { + const char* custom_code) { TFLITE_DCHECK(next_operator_code_id_ <= kMaxOperatorCodes); - operator_codes_[next_operator_code_id_] = - tflite::CreateOperatorCodeDirect(*builder_, op, custom_code, version); + operator_codes_[next_operator_code_id_] = tflite::CreateOperatorCodeDirect( + *builder_, /*deprecated_builtin_code=*/0, custom_code, /*version=*/0, op); next_operator_code_id_++; return next_operator_code_id_ - 1; } @@ -142,29 +166,75 @@ ModelBuilder::Node ModelBuilder::AddNode( return next_operator_id_ - 1; } +void ModelBuilder::AddMetadata(const char* description_string, + const int32_t* metadata_buffer_data, + size_t num_elements) { + metadata_[ModelBuilder::nbr_of_metadata_buffers_] = + CreateMetadata(*builder_, builder_->CreateString(description_string), + 1 + ModelBuilder::nbr_of_metadata_buffers_); + + metadata_buffers_[nbr_of_metadata_buffers_] = tflite::CreateBuffer( + *builder_, builder_->CreateVector((uint8_t*)metadata_buffer_data, + sizeof(uint32_t) * num_elements)); + + ModelBuilder::nbr_of_metadata_buffers_++; +} + const Model* ModelBuilder::BuildModel( std::initializer_list inputs, - std::initializer_list outputs) { + std::initializer_list outputs, + size_t num_subgraph_inputs) { // Model schema requires an empty buffer at idx 0. - constexpr size_t kBufferSize = 1; - const flatbuffers::Offset buffers[kBufferSize] = { - tflite::CreateBuffer(*builder_)}; + size_t buffer_size = 1 + ModelBuilder::nbr_of_metadata_buffers_; + flatbuffers::Offset buffers[kMaxMetadataBuffers]; + buffers[0] = tflite::CreateBuffer(*builder_); + + // Place the metadata buffers first in the buffer since the indices for them + // have already been set in AddMetadata() + for (int i = 1; i < ModelBuilder::nbr_of_metadata_buffers_ + 1; ++i) { + buffers[i] = metadata_buffers_[i - 1]; + } // TFLM only supports single subgraph. constexpr size_t subgraphs_size = 1; + + // Find out number of subgraph inputs. + if (num_subgraph_inputs == 0) { + // This is the default case. + num_subgraph_inputs = inputs.size(); + } else { + // A non-zero value of num_subgraph_inputs means that some of + // the operator input tensors are not subgraph inputs. + TFLITE_DCHECK(num_subgraph_inputs <= inputs.size()); + } + const flatbuffers::Offset subgraphs[subgraphs_size] = { tflite::CreateSubGraph( *builder_, builder_->CreateVector(tensors_, next_tensor_id_), - builder_->CreateVector(inputs.begin(), inputs.size()), + builder_->CreateVector(inputs.begin(), num_subgraph_inputs), builder_->CreateVector(outputs.begin(), outputs.size()), builder_->CreateVector(operators_, next_operator_id_), builder_->CreateString("test_subgraph"))}; - const flatbuffers::Offset model_offset = tflite::CreateModel( - *builder_, 0, - builder_->CreateVector(operator_codes_, next_operator_code_id_), - builder_->CreateVector(subgraphs, subgraphs_size), - builder_->CreateString("teset_model"), - builder_->CreateVector(buffers, kBufferSize)); + + flatbuffers::Offset model_offset; + if (ModelBuilder::nbr_of_metadata_buffers_ > 0) { + model_offset = tflite::CreateModel( + *builder_, 0, + builder_->CreateVector(operator_codes_, next_operator_code_id_), + builder_->CreateVector(subgraphs, subgraphs_size), + builder_->CreateString("teset_model"), + builder_->CreateVector(buffers, buffer_size), 0, + builder_->CreateVector(metadata_, + ModelBuilder::nbr_of_metadata_buffers_)); + } else { + model_offset = tflite::CreateModel( + *builder_, 0, + builder_->CreateVector(operator_codes_, next_operator_code_id_), + builder_->CreateVector(subgraphs, subgraphs_size), + builder_->CreateString("teset_model"), + builder_->CreateVector(buffers, buffer_size)); + } + tflite::FinishModelBuffer(*builder_, model_offset); void* model_pointer = builder_->GetBufferPointer(); const Model* model = flatbuffers::GetRoot(model_pointer); @@ -190,7 +260,7 @@ const Model* BuildSimpleStatefulModel() { ModelBuilder model_builder(fb_builder); const int op_id = - model_builder.RegisterOp(BuiltinOperator_CUSTOM, "simple_stateful_op", 0); + model_builder.RegisterOp(BuiltinOperator_CUSTOM, "simple_stateful_op"); const int input_tensor = model_builder.AddTensor(TensorType_UINT8, {3}); const int median_tensor = model_builder.AddTensor(TensorType_UINT8, {3}); const int invoke_count_tensor = @@ -231,8 +301,7 @@ const Model* BuildSimpleModelWithBranch() { v */ const int op_id = - model_builder.RegisterOp(BuiltinOperator_CUSTOM, "mock_custom", - /* version= */ 0); + model_builder.RegisterOp(BuiltinOperator_CUSTOM, "mock_custom"); const int t0 = model_builder.AddTensor(TensorType_FLOAT32, {2, 2, 3}); const int t1 = model_builder.AddTensor(TensorType_FLOAT32, {2, 2, 3}); const int t2 = model_builder.AddTensor(TensorType_FLOAT32, {2, 2, 3}); @@ -243,6 +312,35 @@ const Model* BuildSimpleModelWithBranch() { return model_builder.BuildModel({t0}, {t3}); } +const Model* BuildModelWithOfflinePlanning(int number_of_tensors, + const int32_t* metadata_buffer, + NodeConnection* node_conn, + int num_conns, + int num_subgraph_inputs) { + using flatbuffers::Offset; + flatbuffers::FlatBufferBuilder* fb_builder = BuilderInstance(); + + ModelBuilder model_builder(fb_builder); + + const int op_id = + model_builder.RegisterOp(BuiltinOperator_CUSTOM, "mock_custom"); + + for (int i = 0; i < number_of_tensors; ++i) { + model_builder.AddTensor(TensorType_FLOAT32, {2, 2, 3}); + } + + for (int i = 0; i < num_conns; ++i) { + model_builder.AddNode(op_id, node_conn[i].input, node_conn[i].output); + } + + model_builder.AddMetadata( + "OfflineMemoryAllocation", metadata_buffer, + number_of_tensors + tflite::testing::kOfflinePlannerHeaderSize); + + return model_builder.BuildModel( + node_conn[0].input, node_conn[num_conns - 1].output, num_subgraph_inputs); +} + const Model* BuildSimpleMockModel() { using flatbuffers::Offset; flatbuffers::FlatBufferBuilder* builder = BuilderInstance(); @@ -306,8 +404,9 @@ const Model* BuildSimpleMockModel() { builder->CreateString("test_subgraph"))}; constexpr size_t operator_codes_size = 1; const Offset operator_codes[operator_codes_size] = { - CreateOperatorCodeDirect(*builder, BuiltinOperator_CUSTOM, "mock_custom", - 0)}; + CreateOperatorCodeDirect(*builder, /*deprecated_builtin_code=*/0, + "mock_custom", + /*version=*/0, BuiltinOperator_CUSTOM)}; const Offset model_offset = CreateModel( *builder, 0, builder->CreateVector(operator_codes, operator_codes_size), builder->CreateVector(subgraphs, subgraphs_size), @@ -455,8 +554,9 @@ const Model* BuildComplexMockModel() { constexpr size_t operator_codes_size = 1; const Offset operator_codes[operator_codes_size] = { - CreateOperatorCodeDirect(*builder, BuiltinOperator_CUSTOM, "mock_custom", - 0)}; + CreateOperatorCodeDirect(*builder, /*deprecated_builtin_code=*/0, + "mock_custom", + /*version=*/0, BuiltinOperator_CUSTOM)}; const Offset model_offset = CreateModel( *builder, 0, builder->CreateVector(operator_codes, operator_codes_size), @@ -472,6 +572,147 @@ const Model* BuildComplexMockModel() { } // namespace +const TfLiteRegistration* SimpleStatefulOp::getRegistration() { + return GetMutableRegistration(); +} + +TfLiteRegistration* SimpleStatefulOp::GetMutableRegistration() { + static TfLiteRegistration r; + r.init = Init; + r.prepare = Prepare; + r.invoke = Invoke; + return &r; +} + +void* SimpleStatefulOp::Init(TfLiteContext* context, const char* buffer, + size_t length) { + TFLITE_DCHECK(context->AllocateBufferForEval == nullptr); + TFLITE_DCHECK(context->GetScratchBuffer == nullptr); + TFLITE_DCHECK(context->RequestScratchBufferInArena == nullptr); + + void* raw = context->AllocatePersistentBuffer(context, sizeof(OpData)); + OpData* data = reinterpret_cast(raw); + *data = {}; + return raw; +} + +TfLiteStatus SimpleStatefulOp::Prepare(TfLiteContext* context, + TfLiteNode* node) { + OpData* data = reinterpret_cast(node->user_data); + + // Make sure that the input is in uint8_t with at least 1 data entry. + const TfLiteTensor* input; + TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input)); + if (input->type != kTfLiteUInt8) return kTfLiteError; + if (NumElements(input->dims) == 0) return kTfLiteError; + + // Allocate a temporary buffer with the same size of input for sorting. + TF_LITE_ENSURE_STATUS(context->RequestScratchBufferInArena( + context, sizeof(uint8_t) * NumElements(input->dims), + &data->sorting_buffer)); + // We can interleave scratch / persistent buffer allocation. + data->invoke_count = reinterpret_cast( + context->AllocatePersistentBuffer(context, sizeof(int))); + *data->invoke_count = 0; + + return kTfLiteOk; +} + +TfLiteStatus SimpleStatefulOp::Invoke(TfLiteContext* context, + TfLiteNode* node) { + OpData* data = reinterpret_cast(node->user_data); + *data->invoke_count += 1; + + const TfLiteTensor* input; + TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input)); + const uint8_t* input_data = GetTensorData(input); + int size = NumElements(input->dims); + + uint8_t* sorting_buffer = reinterpret_cast( + context->GetScratchBuffer(context, data->sorting_buffer)); + // Copy inputs data to the sorting buffer. We don't want to mutate the input + // tensor as it might be used by a another node. + for (int i = 0; i < size; i++) { + sorting_buffer[i] = input_data[i]; + } + + // In place insertion sort on `sorting_buffer`. + for (int i = 1; i < size; i++) { + for (int j = i; j > 0 && sorting_buffer[j] < sorting_buffer[j - 1]; j--) { + std::swap(sorting_buffer[j], sorting_buffer[j - 1]); + } + } + + TfLiteTensor* median; + TF_LITE_ENSURE_OK(context, + GetOutputSafe(context, node, kMedianTensor, &median)); + uint8_t* median_data = GetTensorData(median); + TfLiteTensor* invoke_count; + TF_LITE_ENSURE_OK(context, + GetOutputSafe(context, node, kInvokeCount, &invoke_count)); + int32_t* invoke_count_data = GetTensorData(invoke_count); + + median_data[0] = sorting_buffer[size / 2]; + invoke_count_data[0] = *data->invoke_count; + return kTfLiteOk; +} + +const TfLiteRegistration* MockCustom::getRegistration() { + return GetMutableRegistration(); +} + +TfLiteRegistration* MockCustom::GetMutableRegistration() { + static TfLiteRegistration r; + r.init = Init; + r.prepare = Prepare; + r.invoke = Invoke; + r.free = Free; + return &r; +} + +void* MockCustom::Init(TfLiteContext* context, const char* buffer, + size_t length) { + // We don't support delegate in TFL micro. This is a weak check to test if + // context struct being zero-initialized. + TFLITE_DCHECK(context->ReplaceNodeSubsetsWithDelegateKernels == nullptr); + freed_ = false; + // Do nothing. + return nullptr; +} + +void MockCustom::Free(TfLiteContext* context, void* buffer) { freed_ = true; } + +TfLiteStatus MockCustom::Prepare(TfLiteContext* context, TfLiteNode* node) { + return kTfLiteOk; +} + +TfLiteStatus MockCustom::Invoke(TfLiteContext* context, TfLiteNode* node) { + const TfLiteTensor* input; + TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &input)); + const int32_t* input_data = input->data.i32; + const TfLiteTensor* weight; + TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 1, &weight)); + const uint8_t* weight_data = weight->data.uint8; + TfLiteTensor* output; + TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output)); + int32_t* output_data = output->data.i32; + output_data[0] = + 0; // Catch output tensor sharing memory with an input tensor + output_data[0] = input_data[0] + weight_data[0]; + return kTfLiteOk; +} + +bool MockCustom::freed_ = false; + +AllOpsResolver GetOpResolver() { + AllOpsResolver op_resolver; + op_resolver.AddCustom("mock_custom", MockCustom::GetMutableRegistration()); + op_resolver.AddCustom("simple_stateful_op", + SimpleStatefulOp::GetMutableRegistration()); + + return op_resolver; +} + const Model* GetSimpleMockModel() { static Model* model = nullptr; if (!model) { @@ -496,6 +737,16 @@ const Model* GetSimpleModelWithBranch() { return model; } +const Model* GetModelWithOfflinePlanning(int num_tensors, + const int32_t* metadata_buffer, + NodeConnection* node_conn, + int num_conns, + int num_subgraph_inputs) { + const Model* model = BuildModelWithOfflinePlanning( + num_tensors, metadata_buffer, node_conn, num_conns, num_subgraph_inputs); + return model; +} + const Model* GetSimpleStatefulModel() { static Model* model = nullptr; if (!model) { @@ -592,11 +843,13 @@ int TestStrcmp(const char* a, const char* b) { // Wrapper to forward kernel errors to the interpreter's error reporter. void ReportOpError(struct TfLiteContext* context, const char* format, ...) { +#ifndef TF_LITE_STRIP_ERROR_STRINGS ErrorReporter* error_reporter = static_cast(context->impl_); va_list args; va_start(args, format); TF_LITE_REPORT_ERROR(error_reporter, format, args); va_end(args); +#endif } // Create a TfLiteIntArray from an array of ints. The first element in the @@ -616,122 +869,27 @@ TfLiteFloatArray* FloatArrayFromFloats(const float* floats) { return reinterpret_cast(const_cast(floats)); } -TfLiteTensor CreateTensor(TfLiteIntArray* dims, const char* name, - bool is_variable) { - TfLiteTensor result; - result.dims = dims; - result.name = name; - result.params = {}; - result.quantization = {kTfLiteNoQuantization, nullptr}; - result.is_variable = is_variable; - result.allocation_type = kTfLiteMemNone; - result.allocation = nullptr; - return result; -} - -TfLiteTensor CreateFloatTensor(const float* data, TfLiteIntArray* dims, - const char* name, bool is_variable) { - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteFloat32; - result.data.f = const_cast(data); - result.bytes = ElementCount(*dims) * sizeof(float); - return result; -} - -void PopulateFloatTensor(TfLiteTensor* tensor, float* begin, float* end) { - float* p = begin; - float* v = tensor->data.f; - while (p != end) { - *v++ = *p++; - } -} - -TfLiteTensor CreateBoolTensor(const bool* data, TfLiteIntArray* dims, - const char* name, bool is_variable) { - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteBool; - result.data.b = const_cast(data); - result.bytes = ElementCount(*dims) * sizeof(bool); - return result; -} - -TfLiteTensor CreateInt32Tensor(const int32_t* data, TfLiteIntArray* dims, - const char* name, bool is_variable) { - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteInt32; - result.data.i32 = const_cast(data); - result.bytes = ElementCount(*dims) * sizeof(int32_t); - return result; -} - -TfLiteTensor CreateQuantizedTensor(const uint8_t* data, TfLiteIntArray* dims, - float scale, int zero_point, - const char* name, bool is_variable) { - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteUInt8; - result.data.uint8 = const_cast(data); - result.params = {scale, zero_point}; - result.quantization = {kTfLiteAffineQuantization, nullptr}; - result.bytes = ElementCount(*dims) * sizeof(uint8_t); - return result; -} - -TfLiteTensor CreateQuantizedTensor(const int8_t* data, TfLiteIntArray* dims, - float scale, int zero_point, - const char* name, bool is_variable) { - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteInt8; - result.data.int8 = const_cast(data); - result.params = {scale, zero_point}; - result.quantization = {kTfLiteAffineQuantization, nullptr}; - result.bytes = ElementCount(*dims) * sizeof(int8_t); - return result; -} - -TfLiteTensor CreateQuantizedTensor(const int16_t* data, TfLiteIntArray* dims, - float scale, int zero_point, - const char* name, bool is_variable) { - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteInt16; - result.data.i16 = const_cast(data); - result.params = {scale, zero_point}; - result.quantization = {kTfLiteAffineQuantization, nullptr}; - result.bytes = ElementCount(*dims) * sizeof(int16_t); - return result; -} - -TfLiteTensor CreateQuantized32Tensor(const int32_t* data, TfLiteIntArray* dims, - float scale, const char* name, - bool is_variable) { - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteInt32; - result.data.i32 = const_cast(data); - // Quantized int32 tensors always have a zero point of 0, since the range of - // int32 values is large, and because zero point costs extra cycles during - // processing. - result.params = {scale, 0}; - result.quantization = {kTfLiteAffineQuantization, nullptr}; - result.bytes = ElementCount(*dims) * sizeof(int32_t); - return result; -} - TfLiteTensor CreateQuantizedBiasTensor(const float* data, int32_t* quantized, TfLiteIntArray* dims, float input_scale, - float weights_scale, const char* name, - bool is_variable) { + float weights_scale, bool is_variable) { float bias_scale = input_scale * weights_scale; tflite::SymmetricQuantize(data, quantized, ElementCount(*dims), bias_scale); - return CreateQuantized32Tensor(quantized, dims, bias_scale, name, - is_variable); + + // Quantized int32_t tensors always have a zero point of 0, since the range of + // int32_t values is large, and because zero point costs extra cycles during + // processing. + TfLiteTensor result = + CreateQuantizedTensor(quantized, dims, bias_scale, 0, is_variable); + return result; } -// Quantizes int32 bias tensor with per-channel weights determined by input +// Quantizes int32_t bias tensor with per-channel weights determined by input // scale multiplied by weight scale for each channel. TfLiteTensor CreatePerChannelQuantizedBiasTensor( const float* input, int32_t* quantized, TfLiteIntArray* dims, float input_scale, float* weight_scales, float* scales, int* zero_points, TfLiteAffineQuantization* affine_quant, int quantized_dimension, - const char* name, bool is_variable) { + bool is_variable) { int input_size = ElementCount(*dims); int num_channels = dims->data[quantized_dimension]; // First element is reserved for array length @@ -743,25 +901,22 @@ TfLiteTensor CreatePerChannelQuantizedBiasTensor( zero_points[i + 1] = 0; } - SymmetricPerChannelQuantize(input, quantized, input_size, num_channels, - scales_array); + SymmetricPerChannelQuantize(input, quantized, input_size, + num_channels, scales_array); affine_quant->scale = FloatArrayFromFloats(scales); affine_quant->zero_point = IntArrayFromInts(zero_points); affine_quant->quantized_dimension = quantized_dimension; - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteInt32; - result.data.i32 = const_cast(quantized); + TfLiteTensor result = CreateTensor(quantized, dims, is_variable); result.quantization = {kTfLiteAffineQuantization, affine_quant}; - result.bytes = ElementCount(*dims) * sizeof(int32_t); return result; } TfLiteTensor CreateSymmetricPerChannelQuantizedTensor( const float* input, int8_t* quantized, TfLiteIntArray* dims, float* scales, int* zero_points, TfLiteAffineQuantization* affine_quant, - int quantized_dimension, const char* name, bool is_variable) { + int quantized_dimension, bool is_variable) { int channel_count = dims->data[quantized_dimension]; scales[0] = static_cast(channel_count); zero_points[0] = channel_count; @@ -777,13 +932,18 @@ TfLiteTensor CreateSymmetricPerChannelQuantizedTensor( affine_quant->zero_point = IntArrayFromInts(zero_points); affine_quant->quantized_dimension = quantized_dimension; - TfLiteTensor result = CreateTensor(dims, name, is_variable); - result.type = kTfLiteInt8; - result.data.int8 = const_cast(quantized); + TfLiteTensor result = CreateTensor(quantized, dims, is_variable); result.quantization = {kTfLiteAffineQuantization, affine_quant}; - result.bytes = ElementCount(*dims) * sizeof(int8_t); return result; } +size_t GetModelTensorCount(const Model* model) { + auto* subgraphs = model->subgraphs(); + if (subgraphs) { + return (*subgraphs)[0]->tensors()->size(); + } + return 0; +} + } // namespace testing } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/micro/test_helpers.h b/code/lib/tfmicro/tensorflow/lite/micro/test_helpers.h index 010e1f9e..1db0d81f 100644 --- a/code/lib/tfmicro/tensorflow/lite/micro/test_helpers.h +++ b/code/lib/tfmicro/tensorflow/lite/micro/test_helpers.h @@ -18,15 +18,67 @@ limitations under the License. // Useful functions for writing tests. +#include +#include + +#include "flatbuffers/flatbuffers.h" // from @flatbuffers +#include "tensorflow/lite//kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/all_ops_resolver.h" #include "tensorflow/lite/micro/micro_utils.h" +#include "tensorflow/lite/portable_type_to_tflitetype.h" #include "tensorflow/lite/schema/schema_generated.h" namespace tflite { namespace testing { +constexpr int kOfflinePlannerHeaderSize = 3; + +struct NodeConnection_ { + std::initializer_list input; + std::initializer_list output; +}; +typedef struct NodeConnection_ NodeConnection; + +// A simple operator that returns the median of the input with the number of +// times the kernel was invoked. The implementation below is deliberately +// complicated, just to demonstrate how kernel memory planning works. +class SimpleStatefulOp { + static constexpr int kBufferNotAllocated = 0; + // Inputs: + static constexpr int kInputTensor = 0; + // Outputs: + static constexpr int kMedianTensor = 0; + static constexpr int kInvokeCount = 1; + struct OpData { + int* invoke_count = nullptr; + int sorting_buffer = kBufferNotAllocated; + }; + + public: + static const TfLiteRegistration* getRegistration(); + static TfLiteRegistration* GetMutableRegistration(); + static void* Init(TfLiteContext* context, const char* buffer, size_t length); + static TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node); + static TfLiteStatus Invoke(TfLiteContext* context, TfLiteNode* node); +}; + +class MockCustom { + public: + static const TfLiteRegistration* getRegistration(); + static TfLiteRegistration* GetMutableRegistration(); + static void* Init(TfLiteContext* context, const char* buffer, size_t length); + static void Free(TfLiteContext* context, void* buffer); + static TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node); + static TfLiteStatus Invoke(TfLiteContext* context, TfLiteNode* node); + + static bool freed_; +}; + +// Returns an Op Resolver that can be used in the testing code. +AllOpsResolver GetOpResolver(); + // Returns a simple example flatbuffer TensorFlow Lite model. Contains 1 input, // 1 layer of weights, 1 output Tensor, and 1 operator. const Model* GetSimpleMockModel(); @@ -38,6 +90,24 @@ const Model* GetComplexMockModel(); // Returns a simple flatbuffer model with two branches. const Model* GetSimpleModelWithBranch(); +// Returns a simple flatbuffer model with offline planned tensors +// @param[in] num_tensors Number of tensors in the model. +// @param[in] metadata_buffer Metadata for offline planner. +// @param[in] node_con List of connections, i.e. operators +// in the model. +// @param[in] num_conns Number of connections. +// @param[in] num_subgraph_inputs How many of the input tensors are in +// the subgraph inputs. The default value +// of 0 means all of the input tensors +// are in the subgraph input list. There +// must be at least 1 input tensor in the +// subgraph input list. +const Model* GetModelWithOfflinePlanning(int num_tensors, + const int32_t* metadata_buffer, + NodeConnection* node_conn, + int num_conns, + int num_subgraph_inputs = 0); + // Returns a flatbuffer model with `simple_stateful_op` const Model* GetSimpleStatefulModel(); @@ -72,57 +142,80 @@ TfLiteIntArray* IntArrayFromInts(const int* int_array); // supplied array must be the size of the array expressed as a float. TfLiteFloatArray* FloatArrayFromFloats(const float* floats); -TfLiteTensor CreateFloatTensor(const float* data, TfLiteIntArray* dims, - const char* name, bool is_variable = false); +template +TfLiteTensor CreateTensor(const T* data, TfLiteIntArray* dims, + const bool is_variable = false) { + TfLiteTensor result; + result.dims = dims; + result.params = {}; + result.quantization = {kTfLiteNoQuantization, nullptr}; + result.is_variable = is_variable; + result.allocation_type = kTfLiteMemNone; + result.type = typeToTfLiteType(); + // Const cast is used to allow passing in const and non-const arrays within a + // single CreateTensor method. A Const array should be used for immutable + // input tensors and non-const array should be used for mutable and output + // tensors. + result.data.data = const_cast(data); + result.quantization = {kTfLiteAffineQuantization, nullptr}; + result.bytes = ElementCount(*dims) * sizeof(T); + return result; +} -void PopulateFloatTensor(TfLiteTensor* tensor, float* begin, float* end); - -TfLiteTensor CreateBoolTensor(const bool* data, TfLiteIntArray* dims, - const char* name, bool is_variable = false); - -TfLiteTensor CreateInt32Tensor(const int32_t*, TfLiteIntArray* dims, - const char* name, bool is_variable = false); - -TfLiteTensor CreateQuantizedTensor(const uint8_t* data, TfLiteIntArray* dims, - float scale, int zero_point, - const char* name, bool is_variable = false); - -TfLiteTensor CreateQuantizedTensor(const int8_t* data, TfLiteIntArray* dims, - float scale, int zero_point, - const char* name, bool is_variable = false); - -TfLiteTensor CreateQuantizedTensor(const int16_t* data, TfLiteIntArray* dims, - float scale, int zero_point, - const char* name, bool is_variable = false); +template +TfLiteTensor CreateQuantizedTensor(const T* data, TfLiteIntArray* dims, + const float scale, const int zero_point = 0, + const bool is_variable = false) { + TfLiteTensor result = CreateTensor(data, dims, is_variable); + result.params = {scale, zero_point}; + result.quantization = {kTfLiteAffineQuantization, nullptr}; + return result; +} template TfLiteTensor CreateQuantizedTensor(const float* input, T* quantized, TfLiteIntArray* dims, float scale, - int zero_point, const char* name, - bool is_variable = false) { + int zero_point, bool is_variable = false) { int input_size = ElementCount(*dims); - tflite::AsymmetricQuantize(input, quantized, input_size, scale, zero_point); - return CreateQuantizedTensor(quantized, dims, scale, zero_point, name, - is_variable); + tflite::Quantize(input, quantized, input_size, scale, zero_point); + return CreateQuantizedTensor(quantized, dims, scale, zero_point, is_variable); } TfLiteTensor CreateQuantizedBiasTensor(const float* data, int32_t* quantized, TfLiteIntArray* dims, float input_scale, - float weights_scale, const char* name, + float weights_scale, bool is_variable = false); -// Quantizes int32 bias tensor with per-channel weights determined by input +// Quantizes int32_t bias tensor with per-channel weights determined by input // scale multiplied by weight scale for each channel. TfLiteTensor CreatePerChannelQuantizedBiasTensor( const float* input, int32_t* quantized, TfLiteIntArray* dims, float input_scale, float* weight_scales, float* scales, int* zero_points, TfLiteAffineQuantization* affine_quant, int quantized_dimension, - const char* name, bool is_variable = false); + bool is_variable = false); TfLiteTensor CreateSymmetricPerChannelQuantizedTensor( const float* input, int8_t* quantized, TfLiteIntArray* dims, float* scales, int* zero_points, TfLiteAffineQuantization* affine_quant, - int quantized_dimension, const char* name, bool is_variable = false); + int quantized_dimension, bool is_variable = false); + +// Returns the number of tensors in the default subgraph for a tflite::Model. +size_t GetModelTensorCount(const Model* model); + +// Derives the quantization scaling factor from a min and max range. +template +inline float ScaleFromMinMax(const float min, const float max) { + return (max - min) / + static_cast((std::numeric_limits::max() * 1.0) - + std::numeric_limits::min()); +} + +// Derives the quantization zero point from a min and max range. +template +inline int ZeroPointFromMinMax(const float min, const float max) { + return static_cast(std::numeric_limits::min()) + + static_cast(-min / ScaleFromMinMax(min, max) + 0.5f); +} } // namespace testing } // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/portable_type_to_tflitetype.h b/code/lib/tfmicro/tensorflow/lite/portable_type_to_tflitetype.h new file mode 100644 index 00000000..32423a44 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/portable_type_to_tflitetype.h @@ -0,0 +1,74 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_PORTABLE_TYPE_TO_TFLITETYPE_H_ +#define TENSORFLOW_LITE_PORTABLE_TYPE_TO_TFLITETYPE_H_ + +// Most of the definitions have been moved to this subheader so that Micro +// can include it without relying on , which isn't available on all +// platforms. + +// Arduino build defines abs as a macro here. That is invalid C++, and breaks +// libc++'s header, undefine it. +#ifdef abs +#undef abs +#endif + +#include + +#include "tensorflow/lite/c/common.h" + +namespace tflite { + +// Map statically from a C++ type to a TfLiteType. Used in interpreter for +// safe casts. +// Example: +// typeToTfLiteType() -> kTfLiteBool +template +constexpr TfLiteType typeToTfLiteType() { + return kTfLiteNoType; +} +// Map from TfLiteType to the corresponding C++ type. +// Example: +// TfLiteTypeToType::Type -> bool +template +struct TfLiteTypeToType {}; // Specializations below + +// Template specialization for both typeToTfLiteType and TfLiteTypeToType. +#define MATCH_TYPE_AND_TFLITE_TYPE(CPP_TYPE, TFLITE_TYPE_ENUM) \ + template <> \ + constexpr TfLiteType typeToTfLiteType() { \ + return TFLITE_TYPE_ENUM; \ + } \ + template <> \ + struct TfLiteTypeToType { \ + using Type = CPP_TYPE; \ + } + +// No string mapping is included here, since the TF Lite packed representation +// doesn't correspond to a C++ type well. +MATCH_TYPE_AND_TFLITE_TYPE(int32_t, kTfLiteInt32); +MATCH_TYPE_AND_TFLITE_TYPE(int16_t, kTfLiteInt16); +MATCH_TYPE_AND_TFLITE_TYPE(int64_t, kTfLiteInt64); +MATCH_TYPE_AND_TFLITE_TYPE(float, kTfLiteFloat32); +MATCH_TYPE_AND_TFLITE_TYPE(unsigned char, kTfLiteUInt8); +MATCH_TYPE_AND_TFLITE_TYPE(int8_t, kTfLiteInt8); +MATCH_TYPE_AND_TFLITE_TYPE(bool, kTfLiteBool); +MATCH_TYPE_AND_TFLITE_TYPE(std::complex, kTfLiteComplex64); +MATCH_TYPE_AND_TFLITE_TYPE(std::complex, kTfLiteComplex128); +MATCH_TYPE_AND_TFLITE_TYPE(TfLiteFloat16, kTfLiteFloat16); +MATCH_TYPE_AND_TFLITE_TYPE(double, kTfLiteFloat64); + +} // namespace tflite +#endif // TENSORFLOW_LITE_PORTABLE_TYPE_TO_TFLITETYPE_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/schema/schema_generated.h b/code/lib/tfmicro/tensorflow/lite/schema/schema_generated.h index 39a20c72..84442b86 100644 --- a/code/lib/tfmicro/tensorflow/lite/schema/schema_generated.h +++ b/code/lib/tfmicro/tensorflow/lite/schema/schema_generated.h @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ // automatically generated by the FlatBuffers compiler, do not modify + #ifndef FLATBUFFERS_GENERATED_SCHEMA_TFLITE_H_ #define FLATBUFFERS_GENERATED_SCHEMA_TFLITE_H_ @@ -324,6 +325,9 @@ struct MatrixSetDiagOptionsT; struct IfOptions; struct IfOptionsT; +struct CallOnceOptions; +struct CallOnceOptionsT; + struct WhileOptions; struct WhileOptionsT; @@ -348,6 +352,12 @@ struct SegmentSumOptionsT; struct BatchMatMulOptions; struct BatchMatMulOptionsT; +struct CumsumOptions; +struct CumsumOptionsT; + +struct BroadcastToOptions; +struct BroadcastToOptionsT; + struct OperatorCode; struct OperatorCodeT; @@ -363,6 +373,12 @@ struct BufferT; struct Metadata; struct MetadataT; +struct TensorMap; +struct TensorMapT; + +struct SignatureDef; +struct SignatureDefT; + struct Model; struct ModelT; @@ -378,11 +394,12 @@ enum TensorType { TensorType_COMPLEX64 = 8, TensorType_INT8 = 9, TensorType_FLOAT64 = 10, + TensorType_COMPLEX128 = 11, TensorType_MIN = TensorType_FLOAT32, - TensorType_MAX = TensorType_FLOAT64 + TensorType_MAX = TensorType_COMPLEX128 }; -inline const TensorType (&EnumValuesTensorType())[11] { +inline const TensorType (&EnumValuesTensorType())[12] { static const TensorType values[] = { TensorType_FLOAT32, TensorType_FLOAT16, @@ -394,13 +411,14 @@ inline const TensorType (&EnumValuesTensorType())[11] { TensorType_INT16, TensorType_COMPLEX64, TensorType_INT8, - TensorType_FLOAT64 + TensorType_FLOAT64, + TensorType_COMPLEX128 }; return values; } inline const char * const *EnumNamesTensorType() { - static const char * const names[12] = { + static const char * const names[13] = { "FLOAT32", "FLOAT16", "INT32", @@ -412,13 +430,14 @@ inline const char * const *EnumNamesTensorType() { "COMPLEX64", "INT8", "FLOAT64", + "COMPLEX128", nullptr }; return names; } inline const char *EnumNameTensorType(TensorType e) { - if (flatbuffers::IsOutRange(e, TensorType_FLOAT32, TensorType_FLOAT64)) return ""; + if (flatbuffers::IsOutRange(e, TensorType_FLOAT32, TensorType_COMPLEX128)) return ""; const size_t index = static_cast(e); return EnumNamesTensorType()[index]; } @@ -777,11 +796,15 @@ enum BuiltinOperator { BuiltinOperator_DENSIFY = 124, BuiltinOperator_SEGMENT_SUM = 125, BuiltinOperator_BATCH_MATMUL = 126, + BuiltinOperator_PLACEHOLDER_FOR_GREATER_OP_CODES = 127, + BuiltinOperator_CUMSUM = 128, + BuiltinOperator_CALL_ONCE = 129, + BuiltinOperator_BROADCAST_TO = 130, BuiltinOperator_MIN = BuiltinOperator_ADD, - BuiltinOperator_MAX = BuiltinOperator_BATCH_MATMUL + BuiltinOperator_MAX = BuiltinOperator_BROADCAST_TO }; -inline const BuiltinOperator (&EnumValuesBuiltinOperator())[127] { +inline const BuiltinOperator (&EnumValuesBuiltinOperator())[131] { static const BuiltinOperator values[] = { BuiltinOperator_ADD, BuiltinOperator_AVERAGE_POOL_2D, @@ -909,13 +932,17 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[127] { BuiltinOperator_SELECT_V2, BuiltinOperator_DENSIFY, BuiltinOperator_SEGMENT_SUM, - BuiltinOperator_BATCH_MATMUL + BuiltinOperator_BATCH_MATMUL, + BuiltinOperator_PLACEHOLDER_FOR_GREATER_OP_CODES, + BuiltinOperator_CUMSUM, + BuiltinOperator_CALL_ONCE, + BuiltinOperator_BROADCAST_TO }; return values; } inline const char * const *EnumNamesBuiltinOperator() { - static const char * const names[128] = { + static const char * const names[132] = { "ADD", "AVERAGE_POOL_2D", "CONCATENATION", @@ -1043,13 +1070,17 @@ inline const char * const *EnumNamesBuiltinOperator() { "DENSIFY", "SEGMENT_SUM", "BATCH_MATMUL", + "PLACEHOLDER_FOR_GREATER_OP_CODES", + "CUMSUM", + "CALL_ONCE", + "BROADCAST_TO", nullptr }; return names; } inline const char *EnumNameBuiltinOperator(BuiltinOperator e) { - if (flatbuffers::IsOutRange(e, BuiltinOperator_ADD, BuiltinOperator_BATCH_MATMUL)) return ""; + if (flatbuffers::IsOutRange(e, BuiltinOperator_ADD, BuiltinOperator_BROADCAST_TO)) return ""; const size_t index = static_cast(e); return EnumNamesBuiltinOperator()[index]; } @@ -1157,11 +1188,14 @@ enum BuiltinOptions { BuiltinOptions_DensifyOptions = 99, BuiltinOptions_SegmentSumOptions = 100, BuiltinOptions_BatchMatMulOptions = 101, + BuiltinOptions_CumsumOptions = 102, + BuiltinOptions_CallOnceOptions = 103, + BuiltinOptions_BroadcastToOptions = 104, BuiltinOptions_MIN = BuiltinOptions_NONE, - BuiltinOptions_MAX = BuiltinOptions_BatchMatMulOptions + BuiltinOptions_MAX = BuiltinOptions_BroadcastToOptions }; -inline const BuiltinOptions (&EnumValuesBuiltinOptions())[102] { +inline const BuiltinOptions (&EnumValuesBuiltinOptions())[105] { static const BuiltinOptions values[] = { BuiltinOptions_NONE, BuiltinOptions_Conv2DOptions, @@ -1264,13 +1298,16 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[102] { BuiltinOptions_SelectV2Options, BuiltinOptions_DensifyOptions, BuiltinOptions_SegmentSumOptions, - BuiltinOptions_BatchMatMulOptions + BuiltinOptions_BatchMatMulOptions, + BuiltinOptions_CumsumOptions, + BuiltinOptions_CallOnceOptions, + BuiltinOptions_BroadcastToOptions }; return values; } inline const char * const *EnumNamesBuiltinOptions() { - static const char * const names[103] = { + static const char * const names[106] = { "NONE", "Conv2DOptions", "DepthwiseConv2DOptions", @@ -1373,13 +1410,16 @@ inline const char * const *EnumNamesBuiltinOptions() { "DensifyOptions", "SegmentSumOptions", "BatchMatMulOptions", + "CumsumOptions", + "CallOnceOptions", + "BroadcastToOptions", nullptr }; return names; } inline const char *EnumNameBuiltinOptions(BuiltinOptions e) { - if (flatbuffers::IsOutRange(e, BuiltinOptions_NONE, BuiltinOptions_BatchMatMulOptions)) return ""; + if (flatbuffers::IsOutRange(e, BuiltinOptions_NONE, BuiltinOptions_BroadcastToOptions)) return ""; const size_t index = static_cast(e); return EnumNamesBuiltinOptions()[index]; } @@ -1792,6 +1832,18 @@ template<> struct BuiltinOptionsTraits { static const BuiltinOptions enum_value = BuiltinOptions_BatchMatMulOptions; }; +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_CumsumOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_CallOnceOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_BroadcastToOptions; +}; + struct BuiltinOptionsUnion { BuiltinOptions type; void *value; @@ -2632,6 +2684,30 @@ struct BuiltinOptionsUnion { return type == BuiltinOptions_BatchMatMulOptions ? reinterpret_cast(value) : nullptr; } + tflite::CumsumOptionsT *AsCumsumOptions() { + return type == BuiltinOptions_CumsumOptions ? + reinterpret_cast(value) : nullptr; + } + const tflite::CumsumOptionsT *AsCumsumOptions() const { + return type == BuiltinOptions_CumsumOptions ? + reinterpret_cast(value) : nullptr; + } + tflite::CallOnceOptionsT *AsCallOnceOptions() { + return type == BuiltinOptions_CallOnceOptions ? + reinterpret_cast(value) : nullptr; + } + const tflite::CallOnceOptionsT *AsCallOnceOptions() const { + return type == BuiltinOptions_CallOnceOptions ? + reinterpret_cast(value) : nullptr; + } + tflite::BroadcastToOptionsT *AsBroadcastToOptions() { + return type == BuiltinOptions_BroadcastToOptions ? + reinterpret_cast(value) : nullptr; + } + const tflite::BroadcastToOptionsT *AsBroadcastToOptions() const { + return type == BuiltinOptions_BroadcastToOptions ? + reinterpret_cast(value) : nullptr; + } }; bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type); @@ -4739,22 +4815,29 @@ flatbuffers::Offset CreateConcatenationOptions(flatbuffers struct AddOptionsT : public flatbuffers::NativeTable { typedef AddOptions TableType; tflite::ActivationFunctionType fused_activation_function; + bool pot_scale_int16; AddOptionsT() - : fused_activation_function(tflite::ActivationFunctionType_NONE) { + : fused_activation_function(tflite::ActivationFunctionType_NONE), + pot_scale_int16(true) { } }; struct AddOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef AddOptionsT NativeTableType; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_FUSED_ACTIVATION_FUNCTION = 4 + VT_FUSED_ACTIVATION_FUNCTION = 4, + VT_POT_SCALE_INT16 = 6 }; tflite::ActivationFunctionType fused_activation_function() const { return static_cast(GetField(VT_FUSED_ACTIVATION_FUNCTION, 0)); } + bool pot_scale_int16() const { + return GetField(VT_POT_SCALE_INT16, 1) != 0; + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyField(verifier, VT_FUSED_ACTIVATION_FUNCTION) && + VerifyField(verifier, VT_POT_SCALE_INT16) && verifier.EndTable(); } AddOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -4768,6 +4851,9 @@ struct AddOptionsBuilder { void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { fbb_.AddElement(AddOptions::VT_FUSED_ACTIVATION_FUNCTION, static_cast(fused_activation_function), 0); } + void add_pot_scale_int16(bool pot_scale_int16) { + fbb_.AddElement(AddOptions::VT_POT_SCALE_INT16, static_cast(pot_scale_int16), 1); + } explicit AddOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -4782,8 +4868,10 @@ struct AddOptionsBuilder { inline flatbuffers::Offset CreateAddOptions( flatbuffers::FlatBufferBuilder &_fbb, - tflite::ActivationFunctionType fused_activation_function = tflite::ActivationFunctionType_NONE) { + tflite::ActivationFunctionType fused_activation_function = tflite::ActivationFunctionType_NONE, + bool pot_scale_int16 = true) { AddOptionsBuilder builder_(_fbb); + builder_.add_pot_scale_int16(pot_scale_int16); builder_.add_fused_activation_function(fused_activation_function); return builder_.Finish(); } @@ -5375,22 +5463,29 @@ flatbuffers::Offset CreateResizeBilinearOptions(flatbuffe struct ResizeNearestNeighborOptionsT : public flatbuffers::NativeTable { typedef ResizeNearestNeighborOptions TableType; bool align_corners; + bool half_pixel_centers; ResizeNearestNeighborOptionsT() - : align_corners(false) { + : align_corners(false), + half_pixel_centers(false) { } }; struct ResizeNearestNeighborOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ResizeNearestNeighborOptionsT NativeTableType; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_ALIGN_CORNERS = 4 + VT_ALIGN_CORNERS = 4, + VT_HALF_PIXEL_CENTERS = 6 }; bool align_corners() const { return GetField(VT_ALIGN_CORNERS, 0) != 0; } + bool half_pixel_centers() const { + return GetField(VT_HALF_PIXEL_CENTERS, 0) != 0; + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyField(verifier, VT_ALIGN_CORNERS) && + VerifyField(verifier, VT_HALF_PIXEL_CENTERS) && verifier.EndTable(); } ResizeNearestNeighborOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -5404,6 +5499,9 @@ struct ResizeNearestNeighborOptionsBuilder { void add_align_corners(bool align_corners) { fbb_.AddElement(ResizeNearestNeighborOptions::VT_ALIGN_CORNERS, static_cast(align_corners), 0); } + void add_half_pixel_centers(bool half_pixel_centers) { + fbb_.AddElement(ResizeNearestNeighborOptions::VT_HALF_PIXEL_CENTERS, static_cast(half_pixel_centers), 0); + } explicit ResizeNearestNeighborOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -5418,8 +5516,10 @@ struct ResizeNearestNeighborOptionsBuilder { inline flatbuffers::Offset CreateResizeNearestNeighborOptions( flatbuffers::FlatBufferBuilder &_fbb, - bool align_corners = false) { + bool align_corners = false, + bool half_pixel_centers = false) { ResizeNearestNeighborOptionsBuilder builder_(_fbb); + builder_.add_half_pixel_centers(half_pixel_centers); builder_.add_align_corners(align_corners); return builder_.Finish(); } @@ -5892,22 +5992,29 @@ flatbuffers::Offset CreateDepthToSpaceOptions(flatbuffers:: struct SubOptionsT : public flatbuffers::NativeTable { typedef SubOptions TableType; tflite::ActivationFunctionType fused_activation_function; + bool pot_scale_int16; SubOptionsT() - : fused_activation_function(tflite::ActivationFunctionType_NONE) { + : fused_activation_function(tflite::ActivationFunctionType_NONE), + pot_scale_int16(true) { } }; struct SubOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SubOptionsT NativeTableType; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_FUSED_ACTIVATION_FUNCTION = 4 + VT_FUSED_ACTIVATION_FUNCTION = 4, + VT_POT_SCALE_INT16 = 6 }; tflite::ActivationFunctionType fused_activation_function() const { return static_cast(GetField(VT_FUSED_ACTIVATION_FUNCTION, 0)); } + bool pot_scale_int16() const { + return GetField(VT_POT_SCALE_INT16, 1) != 0; + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyField(verifier, VT_FUSED_ACTIVATION_FUNCTION) && + VerifyField(verifier, VT_POT_SCALE_INT16) && verifier.EndTable(); } SubOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -5921,6 +6028,9 @@ struct SubOptionsBuilder { void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { fbb_.AddElement(SubOptions::VT_FUSED_ACTIVATION_FUNCTION, static_cast(fused_activation_function), 0); } + void add_pot_scale_int16(bool pot_scale_int16) { + fbb_.AddElement(SubOptions::VT_POT_SCALE_INT16, static_cast(pot_scale_int16), 1); + } explicit SubOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -5935,8 +6045,10 @@ struct SubOptionsBuilder { inline flatbuffers::Offset CreateSubOptions( flatbuffers::FlatBufferBuilder &_fbb, - tflite::ActivationFunctionType fused_activation_function = tflite::ActivationFunctionType_NONE) { + tflite::ActivationFunctionType fused_activation_function = tflite::ActivationFunctionType_NONE, + bool pot_scale_int16 = true) { SubOptionsBuilder builder_(_fbb); + builder_.add_pot_scale_int16(pot_scale_int16); builder_.add_fused_activation_function(fused_activation_function); return builder_.Finish(); } @@ -8922,6 +9034,60 @@ inline flatbuffers::Offset CreateIfOptions( flatbuffers::Offset CreateIfOptions(flatbuffers::FlatBufferBuilder &_fbb, const IfOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct CallOnceOptionsT : public flatbuffers::NativeTable { + typedef CallOnceOptions TableType; + int32_t init_subgraph_index; + CallOnceOptionsT() + : init_subgraph_index(0) { + } +}; + +struct CallOnceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef CallOnceOptionsT NativeTableType; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_INIT_SUBGRAPH_INDEX = 4 + }; + int32_t init_subgraph_index() const { + return GetField(VT_INIT_SUBGRAPH_INDEX, 0); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_INIT_SUBGRAPH_INDEX) && + verifier.EndTable(); + } + CallOnceOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(CallOnceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const CallOnceOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct CallOnceOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_init_subgraph_index(int32_t init_subgraph_index) { + fbb_.AddElement(CallOnceOptions::VT_INIT_SUBGRAPH_INDEX, init_subgraph_index, 0); + } + explicit CallOnceOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + CallOnceOptionsBuilder &operator=(const CallOnceOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateCallOnceOptions( + flatbuffers::FlatBufferBuilder &_fbb, + int32_t init_subgraph_index = 0) { + CallOnceOptionsBuilder builder_(_fbb); + builder_.add_init_subgraph_index(init_subgraph_index); + return builder_.Finish(); +} + +flatbuffers::Offset CreateCallOnceOptions(flatbuffers::FlatBufferBuilder &_fbb, const CallOnceOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct WhileOptionsT : public flatbuffers::NativeTable { typedef WhileOptions TableType; int32_t cond_subgraph_index; @@ -9294,26 +9460,135 @@ inline flatbuffers::Offset CreateBatchMatMulOptions( flatbuffers::Offset CreateBatchMatMulOptions(flatbuffers::FlatBufferBuilder &_fbb, const BatchMatMulOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct CumsumOptionsT : public flatbuffers::NativeTable { + typedef CumsumOptions TableType; + bool exclusive; + bool reverse; + CumsumOptionsT() + : exclusive(false), + reverse(false) { + } +}; + +struct CumsumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef CumsumOptionsT NativeTableType; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_EXCLUSIVE = 4, + VT_REVERSE = 6 + }; + bool exclusive() const { + return GetField(VT_EXCLUSIVE, 0) != 0; + } + bool reverse() const { + return GetField(VT_REVERSE, 0) != 0; + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_EXCLUSIVE) && + VerifyField(verifier, VT_REVERSE) && + verifier.EndTable(); + } + CumsumOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(CumsumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const CumsumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct CumsumOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_exclusive(bool exclusive) { + fbb_.AddElement(CumsumOptions::VT_EXCLUSIVE, static_cast(exclusive), 0); + } + void add_reverse(bool reverse) { + fbb_.AddElement(CumsumOptions::VT_REVERSE, static_cast(reverse), 0); + } + explicit CumsumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + CumsumOptionsBuilder &operator=(const CumsumOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateCumsumOptions( + flatbuffers::FlatBufferBuilder &_fbb, + bool exclusive = false, + bool reverse = false) { + CumsumOptionsBuilder builder_(_fbb); + builder_.add_reverse(reverse); + builder_.add_exclusive(exclusive); + return builder_.Finish(); +} + +flatbuffers::Offset CreateCumsumOptions(flatbuffers::FlatBufferBuilder &_fbb, const CumsumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct BroadcastToOptionsT : public flatbuffers::NativeTable { + typedef BroadcastToOptions TableType; + BroadcastToOptionsT() { + } +}; + +struct BroadcastToOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef BroadcastToOptionsT NativeTableType; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + BroadcastToOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(BroadcastToOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const BroadcastToOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct BroadcastToOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit BroadcastToOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + BroadcastToOptionsBuilder &operator=(const BroadcastToOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateBroadcastToOptions( + flatbuffers::FlatBufferBuilder &_fbb) { + BroadcastToOptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateBroadcastToOptions(flatbuffers::FlatBufferBuilder &_fbb, const BroadcastToOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct OperatorCodeT : public flatbuffers::NativeTable { typedef OperatorCode TableType; - tflite::BuiltinOperator builtin_code; + int8_t deprecated_builtin_code; std::string custom_code; int32_t version; + tflite::BuiltinOperator builtin_code; OperatorCodeT() - : builtin_code(tflite::BuiltinOperator_ADD), - version(1) { + : deprecated_builtin_code(0), + version(1), + builtin_code(tflite::BuiltinOperator_ADD) { } }; struct OperatorCode FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef OperatorCodeT NativeTableType; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_BUILTIN_CODE = 4, + VT_DEPRECATED_BUILTIN_CODE = 4, VT_CUSTOM_CODE = 6, - VT_VERSION = 8 + VT_VERSION = 8, + VT_BUILTIN_CODE = 10 }; - tflite::BuiltinOperator builtin_code() const { - return static_cast(GetField(VT_BUILTIN_CODE, 0)); + int8_t deprecated_builtin_code() const { + return GetField(VT_DEPRECATED_BUILTIN_CODE, 0); } const flatbuffers::String *custom_code() const { return GetPointer(VT_CUSTOM_CODE); @@ -9321,12 +9596,16 @@ struct OperatorCode FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { int32_t version() const { return GetField(VT_VERSION, 1); } + tflite::BuiltinOperator builtin_code() const { + return static_cast(GetField(VT_BUILTIN_CODE, 0)); + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && - VerifyField(verifier, VT_BUILTIN_CODE) && + VerifyField(verifier, VT_DEPRECATED_BUILTIN_CODE) && VerifyOffset(verifier, VT_CUSTOM_CODE) && verifier.VerifyString(custom_code()) && VerifyField(verifier, VT_VERSION) && + VerifyField(verifier, VT_BUILTIN_CODE) && verifier.EndTable(); } OperatorCodeT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -9337,8 +9616,8 @@ struct OperatorCode FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { struct OperatorCodeBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; - void add_builtin_code(tflite::BuiltinOperator builtin_code) { - fbb_.AddElement(OperatorCode::VT_BUILTIN_CODE, static_cast(builtin_code), 0); + void add_deprecated_builtin_code(int8_t deprecated_builtin_code) { + fbb_.AddElement(OperatorCode::VT_DEPRECATED_BUILTIN_CODE, deprecated_builtin_code, 0); } void add_custom_code(flatbuffers::Offset custom_code) { fbb_.AddOffset(OperatorCode::VT_CUSTOM_CODE, custom_code); @@ -9346,6 +9625,9 @@ struct OperatorCodeBuilder { void add_version(int32_t version) { fbb_.AddElement(OperatorCode::VT_VERSION, version, 1); } + void add_builtin_code(tflite::BuiltinOperator builtin_code) { + fbb_.AddElement(OperatorCode::VT_BUILTIN_CODE, static_cast(builtin_code), 0); + } explicit OperatorCodeBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -9360,27 +9642,31 @@ struct OperatorCodeBuilder { inline flatbuffers::Offset CreateOperatorCode( flatbuffers::FlatBufferBuilder &_fbb, - tflite::BuiltinOperator builtin_code = tflite::BuiltinOperator_ADD, + int8_t deprecated_builtin_code = 0, flatbuffers::Offset custom_code = 0, - int32_t version = 1) { + int32_t version = 1, + tflite::BuiltinOperator builtin_code = tflite::BuiltinOperator_ADD) { OperatorCodeBuilder builder_(_fbb); + builder_.add_builtin_code(builtin_code); builder_.add_version(version); builder_.add_custom_code(custom_code); - builder_.add_builtin_code(builtin_code); + builder_.add_deprecated_builtin_code(deprecated_builtin_code); return builder_.Finish(); } inline flatbuffers::Offset CreateOperatorCodeDirect( flatbuffers::FlatBufferBuilder &_fbb, - tflite::BuiltinOperator builtin_code = tflite::BuiltinOperator_ADD, + int8_t deprecated_builtin_code = 0, const char *custom_code = nullptr, - int32_t version = 1) { + int32_t version = 1, + tflite::BuiltinOperator builtin_code = tflite::BuiltinOperator_ADD) { auto custom_code__ = custom_code ? _fbb.CreateString(custom_code) : 0; return tflite::CreateOperatorCode( _fbb, - builtin_code, + deprecated_builtin_code, custom_code__, - version); + version, + builtin_code); } flatbuffers::Offset CreateOperatorCode(flatbuffers::FlatBufferBuilder &_fbb, const OperatorCodeT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); @@ -9733,6 +10019,15 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const tflite::BatchMatMulOptions *builtin_options_as_BatchMatMulOptions() const { return builtin_options_type() == tflite::BuiltinOptions_BatchMatMulOptions ? static_cast(builtin_options()) : nullptr; } + const tflite::CumsumOptions *builtin_options_as_CumsumOptions() const { + return builtin_options_type() == tflite::BuiltinOptions_CumsumOptions ? static_cast(builtin_options()) : nullptr; + } + const tflite::CallOnceOptions *builtin_options_as_CallOnceOptions() const { + return builtin_options_type() == tflite::BuiltinOptions_CallOnceOptions ? static_cast(builtin_options()) : nullptr; + } + const tflite::BroadcastToOptions *builtin_options_as_BroadcastToOptions() const { + return builtin_options_type() == tflite::BuiltinOptions_BroadcastToOptions ? static_cast(builtin_options()) : nullptr; + } const flatbuffers::Vector *custom_options() const { return GetPointer *>(VT_CUSTOM_OPTIONS); } @@ -10173,6 +10468,18 @@ template<> inline const tflite::BatchMatMulOptions *Operator::builtin_options_as return builtin_options_as_BatchMatMulOptions(); } +template<> inline const tflite::CumsumOptions *Operator::builtin_options_as() const { + return builtin_options_as_CumsumOptions(); +} + +template<> inline const tflite::CallOnceOptions *Operator::builtin_options_as() const { + return builtin_options_as_CallOnceOptions(); +} + +template<> inline const tflite::BroadcastToOptions *Operator::builtin_options_as() const { + return builtin_options_as_BroadcastToOptions(); +} + struct OperatorBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; @@ -10536,6 +10843,193 @@ inline flatbuffers::Offset CreateMetadataDirect( flatbuffers::Offset CreateMetadata(flatbuffers::FlatBufferBuilder &_fbb, const MetadataT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct TensorMapT : public flatbuffers::NativeTable { + typedef TensorMap TableType; + std::string name; + uint32_t tensor_index; + TensorMapT() + : tensor_index(0) { + } +}; + +struct TensorMap FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef TensorMapT NativeTableType; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_NAME = 4, + VT_TENSOR_INDEX = 6 + }; + const flatbuffers::String *name() const { + return GetPointer(VT_NAME); + } + uint32_t tensor_index() const { + return GetField(VT_TENSOR_INDEX, 0); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_NAME) && + verifier.VerifyString(name()) && + VerifyField(verifier, VT_TENSOR_INDEX) && + verifier.EndTable(); + } + TensorMapT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(TensorMapT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const TensorMapT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct TensorMapBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_name(flatbuffers::Offset name) { + fbb_.AddOffset(TensorMap::VT_NAME, name); + } + void add_tensor_index(uint32_t tensor_index) { + fbb_.AddElement(TensorMap::VT_TENSOR_INDEX, tensor_index, 0); + } + explicit TensorMapBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + TensorMapBuilder &operator=(const TensorMapBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateTensorMap( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset name = 0, + uint32_t tensor_index = 0) { + TensorMapBuilder builder_(_fbb); + builder_.add_tensor_index(tensor_index); + builder_.add_name(name); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateTensorMapDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const char *name = nullptr, + uint32_t tensor_index = 0) { + auto name__ = name ? _fbb.CreateString(name) : 0; + return tflite::CreateTensorMap( + _fbb, + name__, + tensor_index); +} + +flatbuffers::Offset CreateTensorMap(flatbuffers::FlatBufferBuilder &_fbb, const TensorMapT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct SignatureDefT : public flatbuffers::NativeTable { + typedef SignatureDef TableType; + std::vector> inputs; + std::vector> outputs; + std::string method_name; + std::string key; + SignatureDefT() { + } +}; + +struct SignatureDef FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef SignatureDefT NativeTableType; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_INPUTS = 4, + VT_OUTPUTS = 6, + VT_METHOD_NAME = 8, + VT_KEY = 10 + }; + const flatbuffers::Vector> *inputs() const { + return GetPointer> *>(VT_INPUTS); + } + const flatbuffers::Vector> *outputs() const { + return GetPointer> *>(VT_OUTPUTS); + } + const flatbuffers::String *method_name() const { + return GetPointer(VT_METHOD_NAME); + } + const flatbuffers::String *key() const { + return GetPointer(VT_KEY); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_INPUTS) && + verifier.VerifyVector(inputs()) && + verifier.VerifyVectorOfTables(inputs()) && + VerifyOffset(verifier, VT_OUTPUTS) && + verifier.VerifyVector(outputs()) && + verifier.VerifyVectorOfTables(outputs()) && + VerifyOffset(verifier, VT_METHOD_NAME) && + verifier.VerifyString(method_name()) && + VerifyOffset(verifier, VT_KEY) && + verifier.VerifyString(key()) && + verifier.EndTable(); + } + SignatureDefT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(SignatureDefT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const SignatureDefT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct SignatureDefBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_inputs(flatbuffers::Offset>> inputs) { + fbb_.AddOffset(SignatureDef::VT_INPUTS, inputs); + } + void add_outputs(flatbuffers::Offset>> outputs) { + fbb_.AddOffset(SignatureDef::VT_OUTPUTS, outputs); + } + void add_method_name(flatbuffers::Offset method_name) { + fbb_.AddOffset(SignatureDef::VT_METHOD_NAME, method_name); + } + void add_key(flatbuffers::Offset key) { + fbb_.AddOffset(SignatureDef::VT_KEY, key); + } + explicit SignatureDefBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + SignatureDefBuilder &operator=(const SignatureDefBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateSignatureDef( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset>> inputs = 0, + flatbuffers::Offset>> outputs = 0, + flatbuffers::Offset method_name = 0, + flatbuffers::Offset key = 0) { + SignatureDefBuilder builder_(_fbb); + builder_.add_key(key); + builder_.add_method_name(method_name); + builder_.add_outputs(outputs); + builder_.add_inputs(inputs); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateSignatureDefDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const std::vector> *inputs = nullptr, + const std::vector> *outputs = nullptr, + const char *method_name = nullptr, + const char *key = nullptr) { + auto inputs__ = inputs ? _fbb.CreateVector>(*inputs) : 0; + auto outputs__ = outputs ? _fbb.CreateVector>(*outputs) : 0; + auto method_name__ = method_name ? _fbb.CreateString(method_name) : 0; + auto key__ = key ? _fbb.CreateString(key) : 0; + return tflite::CreateSignatureDef( + _fbb, + inputs__, + outputs__, + method_name__, + key__); +} + +flatbuffers::Offset CreateSignatureDef(flatbuffers::FlatBufferBuilder &_fbb, const SignatureDefT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct ModelT : public flatbuffers::NativeTable { typedef Model TableType; uint32_t version; @@ -10545,6 +11039,7 @@ struct ModelT : public flatbuffers::NativeTable { std::vector> buffers; std::vector metadata_buffer; std::vector> metadata; + std::vector> signature_defs; ModelT() : version(0) { } @@ -10559,7 +11054,8 @@ struct Model FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VT_DESCRIPTION = 10, VT_BUFFERS = 12, VT_METADATA_BUFFER = 14, - VT_METADATA = 16 + VT_METADATA = 16, + VT_SIGNATURE_DEFS = 18 }; uint32_t version() const { return GetField(VT_VERSION, 0); @@ -10582,6 +11078,9 @@ struct Model FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const flatbuffers::Vector> *metadata() const { return GetPointer> *>(VT_METADATA); } + const flatbuffers::Vector> *signature_defs() const { + return GetPointer> *>(VT_SIGNATURE_DEFS); + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyField(verifier, VT_VERSION) && @@ -10601,6 +11100,9 @@ struct Model FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VerifyOffset(verifier, VT_METADATA) && verifier.VerifyVector(metadata()) && verifier.VerifyVectorOfTables(metadata()) && + VerifyOffset(verifier, VT_SIGNATURE_DEFS) && + verifier.VerifyVector(signature_defs()) && + verifier.VerifyVectorOfTables(signature_defs()) && verifier.EndTable(); } ModelT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -10632,6 +11134,9 @@ struct ModelBuilder { void add_metadata(flatbuffers::Offset>> metadata) { fbb_.AddOffset(Model::VT_METADATA, metadata); } + void add_signature_defs(flatbuffers::Offset>> signature_defs) { + fbb_.AddOffset(Model::VT_SIGNATURE_DEFS, signature_defs); + } explicit ModelBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -10652,8 +11157,10 @@ inline flatbuffers::Offset CreateModel( flatbuffers::Offset description = 0, flatbuffers::Offset>> buffers = 0, flatbuffers::Offset> metadata_buffer = 0, - flatbuffers::Offset>> metadata = 0) { + flatbuffers::Offset>> metadata = 0, + flatbuffers::Offset>> signature_defs = 0) { ModelBuilder builder_(_fbb); + builder_.add_signature_defs(signature_defs); builder_.add_metadata(metadata); builder_.add_metadata_buffer(metadata_buffer); builder_.add_buffers(buffers); @@ -10672,13 +11179,15 @@ inline flatbuffers::Offset CreateModelDirect( const char *description = nullptr, const std::vector> *buffers = nullptr, const std::vector *metadata_buffer = nullptr, - const std::vector> *metadata = nullptr) { + const std::vector> *metadata = nullptr, + const std::vector> *signature_defs = nullptr) { auto operator_codes__ = operator_codes ? _fbb.CreateVector>(*operator_codes) : 0; auto subgraphs__ = subgraphs ? _fbb.CreateVector>(*subgraphs) : 0; auto description__ = description ? _fbb.CreateString(description) : 0; auto buffers__ = buffers ? _fbb.CreateVector>(*buffers) : 0; auto metadata_buffer__ = metadata_buffer ? _fbb.CreateVector(*metadata_buffer) : 0; auto metadata__ = metadata ? _fbb.CreateVector>(*metadata) : 0; + auto signature_defs__ = signature_defs ? _fbb.CreateVector>(*signature_defs) : 0; return tflite::CreateModel( _fbb, version, @@ -10687,7 +11196,8 @@ inline flatbuffers::Offset CreateModelDirect( description__, buffers__, metadata_buffer__, - metadata__); + metadata__, + signature_defs__); } flatbuffers::Offset CreateModel(flatbuffers::FlatBufferBuilder &_fbb, const ModelT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); @@ -10701,7 +11211,7 @@ inline CustomQuantizationT *CustomQuantization::UnPack(const flatbuffers::resolv inline void CustomQuantization::UnPackTo(CustomQuantizationT *_o, const flatbuffers::resolver_function_t *_resolver) const { (void)_o; (void)_resolver; - { auto _e = custom(); if (_e) { _o->custom.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->custom[_i] = _e->Get(_i); } } } + { auto _e = custom(); if (_e) { _o->custom.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->custom.begin()); } } } inline flatbuffers::Offset CustomQuantization::Pack(flatbuffers::FlatBufferBuilder &_fbb, const CustomQuantizationT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -10825,7 +11335,7 @@ inline Uint8VectorT *Uint8Vector::UnPack(const flatbuffers::resolver_function_t inline void Uint8Vector::UnPackTo(Uint8VectorT *_o, const flatbuffers::resolver_function_t *_resolver) const { (void)_o; (void)_resolver; - { auto _e = values(); if (_e) { _o->values.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->values[_i] = _e->Get(_i); } } } + { auto _e = values(); if (_e) { _o->values.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->values.begin()); } } } inline flatbuffers::Offset Uint8Vector::Pack(flatbuffers::FlatBufferBuilder &_fbb, const Uint8VectorT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -11375,6 +11885,7 @@ inline void AddOptions::UnPackTo(AddOptionsT *_o, const flatbuffers::resolver_fu (void)_o; (void)_resolver; { auto _e = fused_activation_function(); _o->fused_activation_function = _e; } + { auto _e = pot_scale_int16(); _o->pot_scale_int16 = _e; } } inline flatbuffers::Offset AddOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const AddOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -11386,9 +11897,11 @@ inline flatbuffers::Offset CreateAddOptions(flatbuffers::FlatBufferB (void)_o; struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const AddOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; auto _fused_activation_function = _o->fused_activation_function; + auto _pot_scale_int16 = _o->pot_scale_int16; return tflite::CreateAddOptions( _fbb, - _fused_activation_function); + _fused_activation_function, + _pot_scale_int16); } inline MulOptionsT *MulOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { @@ -11634,6 +12147,7 @@ inline void ResizeNearestNeighborOptions::UnPackTo(ResizeNearestNeighborOptionsT (void)_o; (void)_resolver; { auto _e = align_corners(); _o->align_corners = _e; } + { auto _e = half_pixel_centers(); _o->half_pixel_centers = _e; } } inline flatbuffers::Offset ResizeNearestNeighborOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const ResizeNearestNeighborOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -11645,9 +12159,11 @@ inline flatbuffers::Offset CreateResizeNearestNeig (void)_o; struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const ResizeNearestNeighborOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; auto _align_corners = _o->align_corners; + auto _half_pixel_centers = _o->half_pixel_centers; return tflite::CreateResizeNearestNeighborOptions( _fbb, - _align_corners); + _align_corners, + _half_pixel_centers); } inline CallOptionsT *CallOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { @@ -11888,6 +12404,7 @@ inline void SubOptions::UnPackTo(SubOptionsT *_o, const flatbuffers::resolver_fu (void)_o; (void)_resolver; { auto _e = fused_activation_function(); _o->fused_activation_function = _e; } + { auto _e = pot_scale_int16(); _o->pot_scale_int16 = _e; } } inline flatbuffers::Offset SubOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const SubOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -11899,9 +12416,11 @@ inline flatbuffers::Offset CreateSubOptions(flatbuffers::FlatBufferB (void)_o; struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const SubOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; auto _fused_activation_function = _o->fused_activation_function; + auto _pot_scale_int16 = _o->pot_scale_int16; return tflite::CreateSubOptions( _fbb, - _fused_activation_function); + _fused_activation_function, + _pot_scale_int16); } inline DivOptionsT *DivOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { @@ -13441,6 +13960,32 @@ inline flatbuffers::Offset CreateIfOptions(flatbuffers::FlatBufferBui _else_subgraph_index); } +inline CallOnceOptionsT *CallOnceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new CallOnceOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void CallOnceOptions::UnPackTo(CallOnceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = init_subgraph_index(); _o->init_subgraph_index = _e; } +} + +inline flatbuffers::Offset CallOnceOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const CallOnceOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateCallOnceOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateCallOnceOptions(flatbuffers::FlatBufferBuilder &_fbb, const CallOnceOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const CallOnceOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _init_subgraph_index = _o->init_subgraph_index; + return tflite::CreateCallOnceOptions( + _fbb, + _init_subgraph_index); +} + inline WhileOptionsT *WhileOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { auto _o = new WhileOptionsT(); UnPackTo(_o, _resolver); @@ -13637,6 +14182,58 @@ inline flatbuffers::Offset CreateBatchMatMulOptions(flatbuff _adj_y); } +inline CumsumOptionsT *CumsumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new CumsumOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void CumsumOptions::UnPackTo(CumsumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = exclusive(); _o->exclusive = _e; } + { auto _e = reverse(); _o->reverse = _e; } +} + +inline flatbuffers::Offset CumsumOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const CumsumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateCumsumOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateCumsumOptions(flatbuffers::FlatBufferBuilder &_fbb, const CumsumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const CumsumOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _exclusive = _o->exclusive; + auto _reverse = _o->reverse; + return tflite::CreateCumsumOptions( + _fbb, + _exclusive, + _reverse); +} + +inline BroadcastToOptionsT *BroadcastToOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new BroadcastToOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void BroadcastToOptions::UnPackTo(BroadcastToOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset BroadcastToOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const BroadcastToOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateBroadcastToOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateBroadcastToOptions(flatbuffers::FlatBufferBuilder &_fbb, const BroadcastToOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const BroadcastToOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateBroadcastToOptions( + _fbb); +} + inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const { auto _o = new OperatorCodeT(); UnPackTo(_o, _resolver); @@ -13646,9 +14243,10 @@ inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_ inline void OperatorCode::UnPackTo(OperatorCodeT *_o, const flatbuffers::resolver_function_t *_resolver) const { (void)_o; (void)_resolver; - { auto _e = builtin_code(); _o->builtin_code = _e; } + { auto _e = deprecated_builtin_code(); _o->deprecated_builtin_code = _e; } { auto _e = custom_code(); if (_e) _o->custom_code = _e->str(); } { auto _e = version(); _o->version = _e; } + { auto _e = builtin_code(); _o->builtin_code = _e; } } inline flatbuffers::Offset OperatorCode::Pack(flatbuffers::FlatBufferBuilder &_fbb, const OperatorCodeT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -13659,14 +14257,16 @@ inline flatbuffers::Offset CreateOperatorCode(flatbuffers::FlatBuf (void)_rehasher; (void)_o; struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const OperatorCodeT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; - auto _builtin_code = _o->builtin_code; + auto _deprecated_builtin_code = _o->deprecated_builtin_code; auto _custom_code = _o->custom_code.empty() ? 0 : _fbb.CreateString(_o->custom_code); auto _version = _o->version; + auto _builtin_code = _o->builtin_code; return tflite::CreateOperatorCode( _fbb, - _builtin_code, + _deprecated_builtin_code, _custom_code, - _version); + _version, + _builtin_code); } inline OperatorT *Operator::UnPack(const flatbuffers::resolver_function_t *_resolver) const { @@ -13683,7 +14283,7 @@ inline void Operator::UnPackTo(OperatorT *_o, const flatbuffers::resolver_functi { auto _e = outputs(); if (_e) { _o->outputs.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->outputs[_i] = _e->Get(_i); } } } { auto _e = builtin_options_type(); _o->builtin_options.type = _e; } { auto _e = builtin_options(); if (_e) _o->builtin_options.value = tflite::BuiltinOptionsUnion::UnPack(_e, builtin_options_type(), _resolver); } - { auto _e = custom_options(); if (_e) { _o->custom_options.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->custom_options[_i] = _e->Get(_i); } } } + { auto _e = custom_options(); if (_e) { _o->custom_options.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->custom_options.begin()); } } { auto _e = custom_options_format(); _o->custom_options_format = _e; } { auto _e = mutating_variable_inputs(); if (_e) { _o->mutating_variable_inputs.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->mutating_variable_inputs[_i] = _e->Get(_i) != 0; } } } { auto _e = intermediates(); if (_e) { _o->intermediates.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->intermediates[_i] = _e->Get(_i); } } } @@ -13766,7 +14366,7 @@ inline BufferT *Buffer::UnPack(const flatbuffers::resolver_function_t *_resolver inline void Buffer::UnPackTo(BufferT *_o, const flatbuffers::resolver_function_t *_resolver) const { (void)_o; (void)_resolver; - { auto _e = data(); if (_e) { _o->data.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->data[_i] = _e->Get(_i); } } } + { auto _e = data(); if (_e) { _o->data.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->data.begin()); } } } inline flatbuffers::Offset Buffer::Pack(flatbuffers::FlatBufferBuilder &_fbb, const BufferT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -13813,6 +14413,70 @@ inline flatbuffers::Offset CreateMetadata(flatbuffers::FlatBufferBuild _buffer); } +inline TensorMapT *TensorMap::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new TensorMapT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void TensorMap::UnPackTo(TensorMapT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = name(); if (_e) _o->name = _e->str(); } + { auto _e = tensor_index(); _o->tensor_index = _e; } +} + +inline flatbuffers::Offset TensorMap::Pack(flatbuffers::FlatBufferBuilder &_fbb, const TensorMapT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateTensorMap(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateTensorMap(flatbuffers::FlatBufferBuilder &_fbb, const TensorMapT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const TensorMapT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _name = _o->name.empty() ? 0 : _fbb.CreateString(_o->name); + auto _tensor_index = _o->tensor_index; + return tflite::CreateTensorMap( + _fbb, + _name, + _tensor_index); +} + +inline SignatureDefT *SignatureDef::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new SignatureDefT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void SignatureDef::UnPackTo(SignatureDefT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = inputs(); if (_e) { _o->inputs.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->inputs[_i] = std::unique_ptr(_e->Get(_i)->UnPack(_resolver)); } } } + { auto _e = outputs(); if (_e) { _o->outputs.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->outputs[_i] = std::unique_ptr(_e->Get(_i)->UnPack(_resolver)); } } } + { auto _e = method_name(); if (_e) _o->method_name = _e->str(); } + { auto _e = key(); if (_e) _o->key = _e->str(); } +} + +inline flatbuffers::Offset SignatureDef::Pack(flatbuffers::FlatBufferBuilder &_fbb, const SignatureDefT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateSignatureDef(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateSignatureDef(flatbuffers::FlatBufferBuilder &_fbb, const SignatureDefT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const SignatureDefT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _inputs = _o->inputs.size() ? _fbb.CreateVector> (_o->inputs.size(), [](size_t i, _VectorArgs *__va) { return CreateTensorMap(*__va->__fbb, __va->__o->inputs[i].get(), __va->__rehasher); }, &_va ) : 0; + auto _outputs = _o->outputs.size() ? _fbb.CreateVector> (_o->outputs.size(), [](size_t i, _VectorArgs *__va) { return CreateTensorMap(*__va->__fbb, __va->__o->outputs[i].get(), __va->__rehasher); }, &_va ) : 0; + auto _method_name = _o->method_name.empty() ? 0 : _fbb.CreateString(_o->method_name); + auto _key = _o->key.empty() ? 0 : _fbb.CreateString(_o->key); + return tflite::CreateSignatureDef( + _fbb, + _inputs, + _outputs, + _method_name, + _key); +} + inline ModelT *Model::UnPack(const flatbuffers::resolver_function_t *_resolver) const { auto _o = new ModelT(); UnPackTo(_o, _resolver); @@ -13829,6 +14493,7 @@ inline void Model::UnPackTo(ModelT *_o, const flatbuffers::resolver_function_t * { auto _e = buffers(); if (_e) { _o->buffers.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->buffers[_i] = std::unique_ptr(_e->Get(_i)->UnPack(_resolver)); } } } { auto _e = metadata_buffer(); if (_e) { _o->metadata_buffer.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->metadata_buffer[_i] = _e->Get(_i); } } } { auto _e = metadata(); if (_e) { _o->metadata.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->metadata[_i] = std::unique_ptr(_e->Get(_i)->UnPack(_resolver)); } } } + { auto _e = signature_defs(); if (_e) { _o->signature_defs.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->signature_defs[_i] = std::unique_ptr(_e->Get(_i)->UnPack(_resolver)); } } } } inline flatbuffers::Offset Model::Pack(flatbuffers::FlatBufferBuilder &_fbb, const ModelT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -13846,6 +14511,7 @@ inline flatbuffers::Offset CreateModel(flatbuffers::FlatBufferBuilder &_f auto _buffers = _o->buffers.size() ? _fbb.CreateVector> (_o->buffers.size(), [](size_t i, _VectorArgs *__va) { return CreateBuffer(*__va->__fbb, __va->__o->buffers[i].get(), __va->__rehasher); }, &_va ) : 0; auto _metadata_buffer = _o->metadata_buffer.size() ? _fbb.CreateVector(_o->metadata_buffer) : 0; auto _metadata = _o->metadata.size() ? _fbb.CreateVector> (_o->metadata.size(), [](size_t i, _VectorArgs *__va) { return CreateMetadata(*__va->__fbb, __va->__o->metadata[i].get(), __va->__rehasher); }, &_va ) : 0; + auto _signature_defs = _o->signature_defs.size() ? _fbb.CreateVector> (_o->signature_defs.size(), [](size_t i, _VectorArgs *__va) { return CreateSignatureDef(*__va->__fbb, __va->__o->signature_defs[i].get(), __va->__rehasher); }, &_va ) : 0; return tflite::CreateModel( _fbb, _version, @@ -13854,7 +14520,8 @@ inline flatbuffers::Offset CreateModel(flatbuffers::FlatBufferBuilder &_f _description, _buffers, _metadata_buffer, - _metadata); + _metadata, + _signature_defs); } inline bool VerifyQuantizationDetails(flatbuffers::Verifier &verifier, const void *obj, QuantizationDetails type) { @@ -14446,6 +15113,18 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } + case BuiltinOptions_CumsumOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_CallOnceOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_BroadcastToOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } default: return true; } } @@ -14868,6 +15547,18 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } + case BuiltinOptions_CumsumOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_CallOnceOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_BroadcastToOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } default: return nullptr; } } @@ -15278,6 +15969,18 @@ inline flatbuffers::Offset BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff auto ptr = reinterpret_cast(value); return CreateBatchMatMulOptions(_fbb, ptr, _rehasher).Union(); } + case BuiltinOptions_CumsumOptions: { + auto ptr = reinterpret_cast(value); + return CreateCumsumOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_CallOnceOptions: { + auto ptr = reinterpret_cast(value); + return CreateCallOnceOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_BroadcastToOptions: { + auto ptr = reinterpret_cast(value); + return CreateBroadcastToOptions(_fbb, ptr, _rehasher).Union(); + } default: return 0; } } @@ -15688,6 +16391,18 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL value = new tflite::BatchMatMulOptionsT(*reinterpret_cast(u.value)); break; } + case BuiltinOptions_CumsumOptions: { + value = new tflite::CumsumOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_CallOnceOptions: { + value = new tflite::CallOnceOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_BroadcastToOptions: { + value = new tflite::BroadcastToOptionsT(*reinterpret_cast(u.value)); + break; + } default: break; } @@ -16200,6 +16915,21 @@ inline void BuiltinOptionsUnion::Reset() { delete ptr; break; } + case BuiltinOptions_CumsumOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_CallOnceOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_BroadcastToOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } default: break; } value = nullptr; diff --git a/code/lib/tfmicro/tensorflow/lite/schema/schema_utils.cc b/code/lib/tfmicro/tensorflow/lite/schema/schema_utils.cc new file mode 100644 index 00000000..fc19290b --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/schema/schema_utils.cc @@ -0,0 +1,62 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/schema/schema_utils.h" + +#include + +#include "tensorflow/lite/kernels/internal/compatibility.h" + +namespace tflite { + +// The following GetBuiltinCode methods are the utility methods for reading +// builtin operatore code, ensuring compatibility issues between v3 and v3a +// schema. Always the maximum value of the two fields always will be the correct +// value as follows: +// +// - Supporting schema version v3 models +// +// The `builtin_code` field is not available in the v3 models. Flatbuffer +// library will feed zero value, which is the default value in the v3a schema. +// The actual builtin operatore code value will exist in the +// `deprecated_builtin_code` field. At the same time, it implies that +// `deprecated_builtin_code` >= `builtin_code` and the maximum value of the two +// fields will be same with `deprecated_builtin_code'. +// +// - Supporting builtin operator codes beyonds 127 +// +// New builtin operators, whose operator code is larger than 127, can not be +// assigned to the `deprecated_builtin_code` field. In such cases, the +// value of the `builtin_code` field should be used for the builtin operator +// code. In the case, the maximum value of the two fields will be the value of +// the `builtin_code` as the right value. + +BuiltinOperator GetBuiltinCode(const OperatorCode* op_code) { + // Caller should guarantee that the given argument value is not a nullptr. + TFLITE_DCHECK(op_code != nullptr); + + return std::max( + op_code->builtin_code(), + static_cast(op_code->deprecated_builtin_code())); +} + +BuiltinOperator GetBuiltinCode(const OperatorCodeT* op_code) { + // Caller should guarantee that the given argument value is not a nullptr. + TFLITE_DCHECK(op_code != nullptr); + + return std::max(op_code->builtin_code, static_cast( + op_code->deprecated_builtin_code)); +} + +} // namespace tflite diff --git a/code/lib/tfmicro/tensorflow/lite/schema/schema_utils.h b/code/lib/tfmicro/tensorflow/lite/schema/schema_utils.h new file mode 100644 index 00000000..453276b9 --- /dev/null +++ b/code/lib/tfmicro/tensorflow/lite/schema/schema_utils.h @@ -0,0 +1,33 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_SCHEMA_SCHEMA_UTILS_H_ +#define TENSORFLOW_LITE_SCHEMA_SCHEMA_UTILS_H_ + +#include "flatbuffers/flatbuffers.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { + +// The following methods are introduced to resolve op builtin code shortage +// problem. The new builtin opreator will be assigned to the extended builtin +// code field in the flatbuffer schema. Those methods helps to hide builtin code +// details. +BuiltinOperator GetBuiltinCode(const OperatorCode *op_code); + +BuiltinOperator GetBuiltinCode(const OperatorCodeT *op_code); + +} // namespace tflite + +#endif // TENSORFLOW_LITE_SCHEMA_SCHEMA_UTILS_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/string_util.h b/code/lib/tfmicro/tensorflow/lite/string_util.h deleted file mode 100644 index 779b1e12..00000000 --- a/code/lib/tfmicro/tensorflow/lite/string_util.h +++ /dev/null @@ -1,103 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Util methods to read and write String tensors. -// String tensors are considered to be char tensor with protocol. -// [0, 3] 4 bytes: N, num of strings in the tensor in little endian. -// [(i+1)*4, (i+1)*4+3] 4 bytes: offset of i-th string in little endian. -// [(N+2)*4, (N+2)*4+3] 4 bytes: length of the whole char buffer. -// [offset(i), offset(i+1) - 1] : content of i-th string. -// Example of a string tensor: -// [ -// 2, 0, 0, 0, # 2 strings. -// 16, 0, 0, 0, # 0-th string starts from index 16. -// 18, 0, 0, 0, # 1-st string starts from index 18. -// 18, 0, 0, 0, # total length of array. -// 'A', 'B', # 0-th string [16..17]: "AB" -// ] # 1-th string, empty -// -// A typical usage: -// In op.Eval(context, node): -// DynamicBuffer buf; -// # Add string "AB" to tensor, string is stored in dynamic buffer. -// buf.AddString("AB", 2); -// # Write content of DynamicBuffer to tensor in format of string tensor -// # described above. -// buf.WriteToTensor(tensor, nullptr) - -#ifndef TENSORFLOW_LITE_STRING_UTIL_H_ -#define TENSORFLOW_LITE_STRING_UTIL_H_ - -#include - -#include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/string_type.h" - -namespace tflite { - -// Convenient structure to store string pointer and length. -typedef struct { - const char* str; - int len; -} StringRef; - -// DynamicBuffer holds temporary buffer that will be used to create a dynamic -// tensor. A typical usage is to initialize a DynamicBuffer object, fill in -// content and call CreateStringTensor in op.Eval(). -class DynamicBuffer { - public: - DynamicBuffer() : offset_({0}) {} - - // Add string to dynamic buffer by resizing the buffer and copying the data. - void AddString(const StringRef& string); - - // Add string to dynamic buffer by resizing the buffer and copying the data. - void AddString(const char* str, size_t len); - - // Join a list of string with separator, and add as a single string to the - // buffer. - void AddJoinedString(const std::vector& strings, char separator); - - // Fill content into a buffer and returns the number of bytes stored. - // The function allocates space for the buffer but does NOT take ownership. - int WriteToBuffer(char** buffer); - - // Fill content into a string tensor, with the given new_shape. The new shape - // must match the number of strings in this object. Caller relinquishes - // ownership of new_shape. If 'new_shape' is nullptr, keep the tensor's - // existing shape. - void WriteToTensor(TfLiteTensor* tensor, TfLiteIntArray* new_shape); - - // Fill content into a string tensor. Set shape to {num_strings}. - void WriteToTensorAsVector(TfLiteTensor* tensor); - - private: - // Data buffer to store contents of strings, not including headers. - std::vector data_; - // Offset of the starting index of each string in data buffer. - std::vector offset_; -}; - -// Return num of strings in a String tensor. -int GetStringCount(const void* raw_buffer); -int GetStringCount(const TfLiteTensor* tensor); - -// Get String pointer and length of index-th string in tensor. -// NOTE: This will not create a copy of string data. -StringRef GetString(const void* raw_buffer, int string_index); -StringRef GetString(const TfLiteTensor* tensor, int string_index); -} // namespace tflite - -#endif // TENSORFLOW_LITE_STRING_UTIL_H_ diff --git a/code/lib/tfmicro/tensorflow/lite/type_to_tflitetype.h b/code/lib/tfmicro/tensorflow/lite/type_to_tflitetype.h deleted file mode 100644 index 84cd54b5..00000000 --- a/code/lib/tfmicro/tensorflow/lite/type_to_tflitetype.h +++ /dev/null @@ -1,82 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#ifndef TENSORFLOW_LITE_TYPE_TO_TFLITETYPE_H_ -#define TENSORFLOW_LITE_TYPE_TO_TFLITETYPE_H_ - -// Arduino build defines abs as a macro here. That is invalid C++, and breaks -// libc++'s header, undefine it. -#ifdef abs -#undef abs -#endif - -#include -#include - -#include "tensorflow/lite/c/common.h" - -namespace tflite { - -// Map statically from a c++ type to a TfLiteType. Used in interpreter for safe -// casts. -template -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteNoType; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteInt32; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteInt16; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteInt64; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteFloat32; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteUInt8; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteInt8; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteBool; -} -template <> -constexpr TfLiteType typeToTfLiteType>() { - return kTfLiteComplex64; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteString; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteFloat16; -} -template <> -constexpr TfLiteType typeToTfLiteType() { - return kTfLiteFloat64; -} -} // namespace tflite -#endif // TENSORFLOW_LITE_TYPE_TO_TFLITETYPE_H_ diff --git a/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/base.h b/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/base.h deleted file mode 100644 index 95573806..00000000 --- a/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/base.h +++ /dev/null @@ -1,398 +0,0 @@ -#ifndef FLATBUFFERS_BASE_H_ -#define FLATBUFFERS_BASE_H_ - -// clang-format off - -// If activate should be declared and included first. -#if defined(FLATBUFFERS_MEMORY_LEAK_TRACKING) && \ - defined(_MSC_VER) && defined(_DEBUG) - // The _CRTDBG_MAP_ALLOC inside will replace - // calloc/free (etc) to its debug version using #define directives. - #define _CRTDBG_MAP_ALLOC - #include - #include - // Replace operator new by trace-enabled version. - #define DEBUG_NEW new(_NORMAL_BLOCK, __FILE__, __LINE__) - #define new DEBUG_NEW -#endif - -#if !defined(FLATBUFFERS_ASSERT) -#include -#define FLATBUFFERS_ASSERT assert -#elif defined(FLATBUFFERS_ASSERT_INCLUDE) -// Include file with forward declaration -#include FLATBUFFERS_ASSERT_INCLUDE -#endif - -#ifndef ARDUINO -#include -#endif - -#include -#include -#include - -#if defined(ARDUINO) && !defined(ARDUINOSTL_M_H) - #include -#else - #include -#endif - -#include -#include -#include -#include -#include -#include -#include - -#ifdef _STLPORT_VERSION - #define FLATBUFFERS_CPP98_STL -#endif -#ifndef FLATBUFFERS_CPP98_STL - #include -#endif - -#include "flatbuffers/stl_emulation.h" - -#if defined(__ICCARM__) -#include -#endif - -// Note the __clang__ check is needed, because clang presents itself -// as an older GNUC compiler (4.2). -// Clang 3.3 and later implement all of the ISO C++ 2011 standard. -// Clang 3.4 and later implement all of the ISO C++ 2014 standard. -// http://clang.llvm.org/cxx_status.html - -// Note the MSVC value '__cplusplus' may be incorrect: -// The '__cplusplus' predefined macro in the MSVC stuck at the value 199711L, -// indicating (erroneously!) that the compiler conformed to the C++98 Standard. -// This value should be correct starting from MSVC2017-15.7-Preview-3. -// The '__cplusplus' will be valid only if MSVC2017-15.7-P3 and the `/Zc:__cplusplus` switch is set. -// Workaround (for details see MSDN): -// Use the _MSC_VER and _MSVC_LANG definition instead of the __cplusplus for compatibility. -// The _MSVC_LANG macro reports the Standard version regardless of the '/Zc:__cplusplus' switch. - -#if defined(__GNUC__) && !defined(__clang__) - #define FLATBUFFERS_GCC (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) -#else - #define FLATBUFFERS_GCC 0 -#endif - -#if defined(__clang__) - #define FLATBUFFERS_CLANG (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) -#else - #define FLATBUFFERS_CLANG 0 -#endif - -/// @cond FLATBUFFERS_INTERNAL -#if __cplusplus <= 199711L && \ - (!defined(_MSC_VER) || _MSC_VER < 1600) && \ - (!defined(__GNUC__) || \ - (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40400)) - #error A C++11 compatible compiler with support for the auto typing is \ - required for FlatBuffers. - #error __cplusplus _MSC_VER __GNUC__ __GNUC_MINOR__ __GNUC_PATCHLEVEL__ -#endif - -#if !defined(__clang__) && \ - defined(__GNUC__) && \ - (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40600) - // Backwards compatibility for g++ 4.4, and 4.5 which don't have the nullptr - // and constexpr keywords. Note the __clang__ check is needed, because clang - // presents itself as an older GNUC compiler. - #ifndef nullptr_t - const class nullptr_t { - public: - template inline operator T*() const { return 0; } - private: - void operator&() const; - } nullptr = {}; - #endif - #ifndef constexpr - #define constexpr const - #endif -#endif - -// The wire format uses a little endian encoding (since that's efficient for -// the common platforms). -#if defined(__s390x__) - #define FLATBUFFERS_LITTLEENDIAN 0 -#endif // __s390x__ -#if !defined(FLATBUFFERS_LITTLEENDIAN) - #if defined(__GNUC__) || defined(__clang__) || defined(__ICCARM__) - #if (defined(__BIG_ENDIAN__) || \ - (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) - #define FLATBUFFERS_LITTLEENDIAN 0 - #else - #define FLATBUFFERS_LITTLEENDIAN 1 - #endif // __BIG_ENDIAN__ - #elif defined(_MSC_VER) - #if defined(_M_PPC) - #define FLATBUFFERS_LITTLEENDIAN 0 - #else - #define FLATBUFFERS_LITTLEENDIAN 1 - #endif - #else - #error Unable to determine endianness, define FLATBUFFERS_LITTLEENDIAN. - #endif -#endif // !defined(FLATBUFFERS_LITTLEENDIAN) - -#define FLATBUFFERS_VERSION_MAJOR 1 -#define FLATBUFFERS_VERSION_MINOR 12 -#define FLATBUFFERS_VERSION_REVISION 0 -#define FLATBUFFERS_STRING_EXPAND(X) #X -#define FLATBUFFERS_STRING(X) FLATBUFFERS_STRING_EXPAND(X) -namespace flatbuffers { - // Returns version as string "MAJOR.MINOR.REVISION". - const char* FLATBUFFERS_VERSION(); -} - -#if (!defined(_MSC_VER) || _MSC_VER > 1600) && \ - (!defined(__GNUC__) || (__GNUC__ * 100 + __GNUC_MINOR__ >= 407)) || \ - defined(__clang__) - #define FLATBUFFERS_FINAL_CLASS final - #define FLATBUFFERS_OVERRIDE override - #define FLATBUFFERS_VTABLE_UNDERLYING_TYPE : flatbuffers::voffset_t -#else - #define FLATBUFFERS_FINAL_CLASS - #define FLATBUFFERS_OVERRIDE - #define FLATBUFFERS_VTABLE_UNDERLYING_TYPE -#endif - -#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && \ - (!defined(__GNUC__) || (__GNUC__ * 100 + __GNUC_MINOR__ >= 406)) || \ - (defined(__cpp_constexpr) && __cpp_constexpr >= 200704) - #define FLATBUFFERS_CONSTEXPR constexpr -#else - #define FLATBUFFERS_CONSTEXPR const -#endif - -#if (defined(__cplusplus) && __cplusplus >= 201402L) || \ - (defined(__cpp_constexpr) && __cpp_constexpr >= 201304) - #define FLATBUFFERS_CONSTEXPR_CPP14 FLATBUFFERS_CONSTEXPR -#else - #define FLATBUFFERS_CONSTEXPR_CPP14 -#endif - -#if (defined(__GXX_EXPERIMENTAL_CXX0X__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 406)) || \ - (defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190023026)) || \ - defined(__clang__) - #define FLATBUFFERS_NOEXCEPT noexcept -#else - #define FLATBUFFERS_NOEXCEPT -#endif - -// NOTE: the FLATBUFFERS_DELETE_FUNC macro may change the access mode to -// private, so be sure to put it at the end or reset access mode explicitly. -#if (!defined(_MSC_VER) || _MSC_FULL_VER >= 180020827) && \ - (!defined(__GNUC__) || (__GNUC__ * 100 + __GNUC_MINOR__ >= 404)) || \ - defined(__clang__) - #define FLATBUFFERS_DELETE_FUNC(func) func = delete; -#else - #define FLATBUFFERS_DELETE_FUNC(func) private: func; -#endif - -#ifndef FLATBUFFERS_HAS_STRING_VIEW - // Only provide flatbuffers::string_view if __has_include can be used - // to detect a header that provides an implementation - #if defined(__has_include) - // Check for std::string_view (in c++17) - #if __has_include() && (__cplusplus >= 201606 || (defined(_HAS_CXX17) && _HAS_CXX17)) - #include - namespace flatbuffers { - typedef std::string_view string_view; - } - #define FLATBUFFERS_HAS_STRING_VIEW 1 - // Check for std::experimental::string_view (in c++14, compiler-dependent) - #elif __has_include() && (__cplusplus >= 201411) - #include - namespace flatbuffers { - typedef std::experimental::string_view string_view; - } - #define FLATBUFFERS_HAS_STRING_VIEW 1 - // Check for absl::string_view - #elif __has_include("absl/strings/string_view.h") - #include "absl/strings/string_view.h" - namespace flatbuffers { - typedef absl::string_view string_view; - } - #define FLATBUFFERS_HAS_STRING_VIEW 1 - #endif - #endif // __has_include -#endif // !FLATBUFFERS_HAS_STRING_VIEW - -#ifndef FLATBUFFERS_HAS_NEW_STRTOD - // Modern (C++11) strtod and strtof functions are available for use. - // 1) nan/inf strings as argument of strtod; - // 2) hex-float as argument of strtod/strtof. - #if (defined(_MSC_VER) && _MSC_VER >= 1900) || \ - (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 409)) || \ - (defined(__clang__)) - #define FLATBUFFERS_HAS_NEW_STRTOD 1 - #endif -#endif // !FLATBUFFERS_HAS_NEW_STRTOD - -#ifndef FLATBUFFERS_LOCALE_INDEPENDENT - // Enable locale independent functions {strtof_l, strtod_l,strtoll_l, strtoull_l}. - // They are part of the POSIX-2008 but not part of the C/C++ standard. - // GCC/Clang have definition (_XOPEN_SOURCE>=700) if POSIX-2008. - #if ((defined(_MSC_VER) && _MSC_VER >= 1800) || \ - (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE>=700))) - #define FLATBUFFERS_LOCALE_INDEPENDENT 1 - #else - #define FLATBUFFERS_LOCALE_INDEPENDENT 0 - #endif -#endif // !FLATBUFFERS_LOCALE_INDEPENDENT - -// Suppress Undefined Behavior Sanitizer (recoverable only). Usage: -// - __supress_ubsan__("undefined") -// - __supress_ubsan__("signed-integer-overflow") -#if defined(__clang__) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >=7)) - #define __supress_ubsan__(type) __attribute__((no_sanitize(type))) -#elif defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 409) - #define __supress_ubsan__(type) __attribute__((no_sanitize_undefined)) -#else - #define __supress_ubsan__(type) -#endif - -// This is constexpr function used for checking compile-time constants. -// Avoid `#pragma warning(disable: 4127) // C4127: expression is constant`. -template FLATBUFFERS_CONSTEXPR inline bool IsConstTrue(T t) { - return !!t; -} - -// Enable C++ attribute [[]] if std:c++17 or higher. -#if ((__cplusplus >= 201703L) \ - || (defined(_MSVC_LANG) && (_MSVC_LANG >= 201703L))) - // All attributes unknown to an implementation are ignored without causing an error. - #define FLATBUFFERS_ATTRIBUTE(attr) [[attr]] - - #define FLATBUFFERS_FALLTHROUGH() [[fallthrough]] -#else - #define FLATBUFFERS_ATTRIBUTE(attr) - - #if FLATBUFFERS_CLANG >= 30800 - #define FLATBUFFERS_FALLTHROUGH() [[clang::fallthrough]] - #elif FLATBUFFERS_GCC >= 70300 - #define FLATBUFFERS_FALLTHROUGH() [[gnu::fallthrough]] - #else - #define FLATBUFFERS_FALLTHROUGH() - #endif -#endif - -/// @endcond - -/// @file -namespace flatbuffers { - -/// @cond FLATBUFFERS_INTERNAL -// Our default offset / size type, 32bit on purpose on 64bit systems. -// Also, using a consistent offset type maintains compatibility of serialized -// offset values between 32bit and 64bit systems. -typedef uint32_t uoffset_t; - -// Signed offsets for references that can go in both directions. -typedef int32_t soffset_t; - -// Offset/index used in v-tables, can be changed to uint8_t in -// format forks to save a bit of space if desired. -typedef uint16_t voffset_t; - -typedef uintmax_t largest_scalar_t; - -// In 32bits, this evaluates to 2GB - 1 -#define FLATBUFFERS_MAX_BUFFER_SIZE ((1ULL << (sizeof(::flatbuffers::soffset_t) * 8 - 1)) - 1) - -// We support aligning the contents of buffers up to this size. -#define FLATBUFFERS_MAX_ALIGNMENT 16 - -#if defined(_MSC_VER) - #pragma warning(push) - #pragma warning(disable: 4127) // C4127: conditional expression is constant -#endif - -template T EndianSwap(T t) { - #if defined(_MSC_VER) - #define FLATBUFFERS_BYTESWAP16 _byteswap_ushort - #define FLATBUFFERS_BYTESWAP32 _byteswap_ulong - #define FLATBUFFERS_BYTESWAP64 _byteswap_uint64 - #elif defined(__ICCARM__) - #define FLATBUFFERS_BYTESWAP16 __REV16 - #define FLATBUFFERS_BYTESWAP32 __REV - #define FLATBUFFERS_BYTESWAP64(x) \ - ((__REV(static_cast(x >> 32U))) | (static_cast(__REV(static_cast(x)))) << 32U) - #else - #if defined(__GNUC__) && __GNUC__ * 100 + __GNUC_MINOR__ < 408 && !defined(__clang__) - // __builtin_bswap16 was missing prior to GCC 4.8. - #define FLATBUFFERS_BYTESWAP16(x) \ - static_cast(__builtin_bswap32(static_cast(x) << 16)) - #else - #define FLATBUFFERS_BYTESWAP16 __builtin_bswap16 - #endif - #define FLATBUFFERS_BYTESWAP32 __builtin_bswap32 - #define FLATBUFFERS_BYTESWAP64 __builtin_bswap64 - #endif - if (sizeof(T) == 1) { // Compile-time if-then's. - return t; - } else if (sizeof(T) == 2) { - union { T t; uint16_t i; } u = { t }; - u.i = FLATBUFFERS_BYTESWAP16(u.i); - return u.t; - } else if (sizeof(T) == 4) { - union { T t; uint32_t i; } u = { t }; - u.i = FLATBUFFERS_BYTESWAP32(u.i); - return u.t; - } else if (sizeof(T) == 8) { - union { T t; uint64_t i; } u = { t }; - u.i = FLATBUFFERS_BYTESWAP64(u.i); - return u.t; - } else { - FLATBUFFERS_ASSERT(0); - return t; - } -} - -#if defined(_MSC_VER) - #pragma warning(pop) -#endif - - -template T EndianScalar(T t) { - #if FLATBUFFERS_LITTLEENDIAN - return t; - #else - return EndianSwap(t); - #endif -} - -template -// UBSAN: C++ aliasing type rules, see std::bit_cast<> for details. -__supress_ubsan__("alignment") -T ReadScalar(const void *p) { - return EndianScalar(*reinterpret_cast(p)); -} - -template -// UBSAN: C++ aliasing type rules, see std::bit_cast<> for details. -__supress_ubsan__("alignment") -void WriteScalar(void *p, T t) { - *reinterpret_cast(p) = EndianScalar(t); -} - -template struct Offset; -template __supress_ubsan__("alignment") void WriteScalar(void *p, Offset t) { - *reinterpret_cast(p) = EndianScalar(t.o); -} - -// Computes how many bytes you'd have to pad to be able to write an -// "scalar_size" scalar if the buffer had grown to "buf_size" (downwards in -// memory). -__supress_ubsan__("unsigned-integer-overflow") -inline size_t PaddingBytes(size_t buf_size, size_t scalar_size) { - return ((~buf_size) + 1) & (scalar_size - 1); -} - -} // namespace flatbuffers -#endif // FLATBUFFERS_BASE_H_ diff --git a/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/flatbuffers.h b/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/flatbuffers.h deleted file mode 100644 index c4dc5bcd..00000000 --- a/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/flatbuffers.h +++ /dev/null @@ -1,2783 +0,0 @@ -/* - * Copyright 2014 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLATBUFFERS_H_ -#define FLATBUFFERS_H_ - -#include "flatbuffers/base.h" - -#if defined(FLATBUFFERS_NAN_DEFAULTS) -# include -#endif - -namespace flatbuffers { -// Generic 'operator==' with conditional specialisations. -// T e - new value of a scalar field. -// T def - default of scalar (is known at compile-time). -template inline bool IsTheSameAs(T e, T def) { return e == def; } - -#if defined(FLATBUFFERS_NAN_DEFAULTS) && \ - defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0) -// Like `operator==(e, def)` with weak NaN if T=(float|double). -template inline bool IsFloatTheSameAs(T e, T def) { - return (e == def) || ((def != def) && (e != e)); -} -template<> inline bool IsTheSameAs(float e, float def) { - return IsFloatTheSameAs(e, def); -} -template<> inline bool IsTheSameAs(double e, double def) { - return IsFloatTheSameAs(e, def); -} -#endif - -// Check 'v' is out of closed range [low; high]. -// Workaround for GCC warning [-Werror=type-limits]: -// comparison is always true due to limited range of data type. -template -inline bool IsOutRange(const T &v, const T &low, const T &high) { - return (v < low) || (high < v); -} - -// Check 'v' is in closed range [low; high]. -template -inline bool IsInRange(const T &v, const T &low, const T &high) { - return !IsOutRange(v, low, high); -} - -// Wrapper for uoffset_t to allow safe template specialization. -// Value is allowed to be 0 to indicate a null object (see e.g. AddOffset). -template struct Offset { - uoffset_t o; - Offset() : o(0) {} - Offset(uoffset_t _o) : o(_o) {} - Offset Union() const { return Offset(o); } - bool IsNull() const { return !o; } -}; - -inline void EndianCheck() { - int endiantest = 1; - // If this fails, see FLATBUFFERS_LITTLEENDIAN above. - FLATBUFFERS_ASSERT(*reinterpret_cast(&endiantest) == - FLATBUFFERS_LITTLEENDIAN); - (void)endiantest; -} - -template FLATBUFFERS_CONSTEXPR size_t AlignOf() { - // clang-format off - #ifdef _MSC_VER - return __alignof(T); - #else - #ifndef alignof - return __alignof__(T); - #else - return alignof(T); - #endif - #endif - // clang-format on -} - -// When we read serialized data from memory, in the case of most scalars, -// we want to just read T, but in the case of Offset, we want to actually -// perform the indirection and return a pointer. -// The template specialization below does just that. -// It is wrapped in a struct since function templates can't overload on the -// return type like this. -// The typedef is for the convenience of callers of this function -// (avoiding the need for a trailing return decltype) -template struct IndirectHelper { - typedef T return_type; - typedef T mutable_return_type; - static const size_t element_stride = sizeof(T); - static return_type Read(const uint8_t *p, uoffset_t i) { - return EndianScalar((reinterpret_cast(p))[i]); - } -}; -template struct IndirectHelper> { - typedef const T *return_type; - typedef T *mutable_return_type; - static const size_t element_stride = sizeof(uoffset_t); - static return_type Read(const uint8_t *p, uoffset_t i) { - p += i * sizeof(uoffset_t); - return reinterpret_cast(p + ReadScalar(p)); - } -}; -template struct IndirectHelper { - typedef const T *return_type; - typedef T *mutable_return_type; - static const size_t element_stride = sizeof(T); - static return_type Read(const uint8_t *p, uoffset_t i) { - return reinterpret_cast(p + i * sizeof(T)); - } -}; - -// An STL compatible iterator implementation for Vector below, effectively -// calling Get() for every element. -template struct VectorIterator { - typedef std::random_access_iterator_tag iterator_category; - typedef IT value_type; - typedef ptrdiff_t difference_type; - typedef IT *pointer; - typedef IT &reference; - - VectorIterator(const uint8_t *data, uoffset_t i) - : data_(data + IndirectHelper::element_stride * i) {} - VectorIterator(const VectorIterator &other) : data_(other.data_) {} - VectorIterator() : data_(nullptr) {} - - VectorIterator &operator=(const VectorIterator &other) { - data_ = other.data_; - return *this; - } - - // clang-format off - #if !defined(FLATBUFFERS_CPP98_STL) - VectorIterator &operator=(VectorIterator &&other) { - data_ = other.data_; - return *this; - } - #endif // !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - - bool operator==(const VectorIterator &other) const { - return data_ == other.data_; - } - - bool operator<(const VectorIterator &other) const { - return data_ < other.data_; - } - - bool operator!=(const VectorIterator &other) const { - return data_ != other.data_; - } - - difference_type operator-(const VectorIterator &other) const { - return (data_ - other.data_) / IndirectHelper::element_stride; - } - - IT operator*() const { return IndirectHelper::Read(data_, 0); } - - IT operator->() const { return IndirectHelper::Read(data_, 0); } - - VectorIterator &operator++() { - data_ += IndirectHelper::element_stride; - return *this; - } - - VectorIterator operator++(int) { - VectorIterator temp(data_, 0); - data_ += IndirectHelper::element_stride; - return temp; - } - - VectorIterator operator+(const uoffset_t &offset) const { - return VectorIterator(data_ + offset * IndirectHelper::element_stride, - 0); - } - - VectorIterator &operator+=(const uoffset_t &offset) { - data_ += offset * IndirectHelper::element_stride; - return *this; - } - - VectorIterator &operator--() { - data_ -= IndirectHelper::element_stride; - return *this; - } - - VectorIterator operator--(int) { - VectorIterator temp(data_, 0); - data_ -= IndirectHelper::element_stride; - return temp; - } - - VectorIterator operator-(const uoffset_t &offset) const { - return VectorIterator(data_ - offset * IndirectHelper::element_stride, - 0); - } - - VectorIterator &operator-=(const uoffset_t &offset) { - data_ -= offset * IndirectHelper::element_stride; - return *this; - } - - private: - const uint8_t *data_; -}; - -template -struct VectorReverseIterator : public std::reverse_iterator { - explicit VectorReverseIterator(Iterator iter) - : std::reverse_iterator(iter) {} - - typename Iterator::value_type operator*() const { - return *(std::reverse_iterator::current); - } - - typename Iterator::value_type operator->() const { - return *(std::reverse_iterator::current); - } -}; - -struct String; - -// This is used as a helper type for accessing vectors. -// Vector::data() assumes the vector elements start after the length field. -template class Vector { - public: - typedef VectorIterator::mutable_return_type> - iterator; - typedef VectorIterator::return_type> - const_iterator; - typedef VectorReverseIterator reverse_iterator; - typedef VectorReverseIterator const_reverse_iterator; - - uoffset_t size() const { return EndianScalar(length_); } - - // Deprecated: use size(). Here for backwards compatibility. - FLATBUFFERS_ATTRIBUTE(deprecated("use size() instead")) - uoffset_t Length() const { return size(); } - - typedef typename IndirectHelper::return_type return_type; - typedef typename IndirectHelper::mutable_return_type mutable_return_type; - - return_type Get(uoffset_t i) const { - FLATBUFFERS_ASSERT(i < size()); - return IndirectHelper::Read(Data(), i); - } - - return_type operator[](uoffset_t i) const { return Get(i); } - - // If this is a Vector of enums, T will be its storage type, not the enum - // type. This function makes it convenient to retrieve value with enum - // type E. - template E GetEnum(uoffset_t i) const { - return static_cast(Get(i)); - } - - // If this a vector of unions, this does the cast for you. There's no check - // to make sure this is the right type! - template const U *GetAs(uoffset_t i) const { - return reinterpret_cast(Get(i)); - } - - // If this a vector of unions, this does the cast for you. There's no check - // to make sure this is actually a string! - const String *GetAsString(uoffset_t i) const { - return reinterpret_cast(Get(i)); - } - - const void *GetStructFromOffset(size_t o) const { - return reinterpret_cast(Data() + o); - } - - iterator begin() { return iterator(Data(), 0); } - const_iterator begin() const { return const_iterator(Data(), 0); } - - iterator end() { return iterator(Data(), size()); } - const_iterator end() const { return const_iterator(Data(), size()); } - - reverse_iterator rbegin() { return reverse_iterator(end() - 1); } - const_reverse_iterator rbegin() const { - return const_reverse_iterator(end() - 1); - } - - reverse_iterator rend() { return reverse_iterator(begin() - 1); } - const_reverse_iterator rend() const { - return const_reverse_iterator(begin() - 1); - } - - const_iterator cbegin() const { return begin(); } - - const_iterator cend() const { return end(); } - - const_reverse_iterator crbegin() const { return rbegin(); } - - const_reverse_iterator crend() const { return rend(); } - - // Change elements if you have a non-const pointer to this object. - // Scalars only. See reflection.h, and the documentation. - void Mutate(uoffset_t i, const T &val) { - FLATBUFFERS_ASSERT(i < size()); - WriteScalar(data() + i, val); - } - - // Change an element of a vector of tables (or strings). - // "val" points to the new table/string, as you can obtain from - // e.g. reflection::AddFlatBuffer(). - void MutateOffset(uoffset_t i, const uint8_t *val) { - FLATBUFFERS_ASSERT(i < size()); - static_assert(sizeof(T) == sizeof(uoffset_t), "Unrelated types"); - WriteScalar(data() + i, - static_cast(val - (Data() + i * sizeof(uoffset_t)))); - } - - // Get a mutable pointer to tables/strings inside this vector. - mutable_return_type GetMutableObject(uoffset_t i) const { - FLATBUFFERS_ASSERT(i < size()); - return const_cast(IndirectHelper::Read(Data(), i)); - } - - // The raw data in little endian format. Use with care. - const uint8_t *Data() const { - return reinterpret_cast(&length_ + 1); - } - - uint8_t *Data() { return reinterpret_cast(&length_ + 1); } - - // Similarly, but typed, much like std::vector::data - const T *data() const { return reinterpret_cast(Data()); } - T *data() { return reinterpret_cast(Data()); } - - template return_type LookupByKey(K key) const { - void *search_result = std::bsearch( - &key, Data(), size(), IndirectHelper::element_stride, KeyCompare); - - if (!search_result) { - return nullptr; // Key not found. - } - - const uint8_t *element = reinterpret_cast(search_result); - - return IndirectHelper::Read(element, 0); - } - - protected: - // This class is only used to access pre-existing data. Don't ever - // try to construct these manually. - Vector(); - - uoffset_t length_; - - private: - // This class is a pointer. Copying will therefore create an invalid object. - // Private and unimplemented copy constructor. - Vector(const Vector &); - Vector &operator=(const Vector &); - - template static int KeyCompare(const void *ap, const void *bp) { - const K *key = reinterpret_cast(ap); - const uint8_t *data = reinterpret_cast(bp); - auto table = IndirectHelper::Read(data, 0); - - // std::bsearch compares with the operands transposed, so we negate the - // result here. - return -table->KeyCompareWithValue(*key); - } -}; - -// Represent a vector much like the template above, but in this case we -// don't know what the element types are (used with reflection.h). -class VectorOfAny { - public: - uoffset_t size() const { return EndianScalar(length_); } - - const uint8_t *Data() const { - return reinterpret_cast(&length_ + 1); - } - uint8_t *Data() { return reinterpret_cast(&length_ + 1); } - - protected: - VectorOfAny(); - - uoffset_t length_; - - private: - VectorOfAny(const VectorOfAny &); - VectorOfAny &operator=(const VectorOfAny &); -}; - -#ifndef FLATBUFFERS_CPP98_STL -template -Vector> *VectorCast(Vector> *ptr) { - static_assert(std::is_base_of::value, "Unrelated types"); - return reinterpret_cast> *>(ptr); -} - -template -const Vector> *VectorCast(const Vector> *ptr) { - static_assert(std::is_base_of::value, "Unrelated types"); - return reinterpret_cast> *>(ptr); -} -#endif - -// Convenient helper function to get the length of any vector, regardless -// of whether it is null or not (the field is not set). -template static inline size_t VectorLength(const Vector *v) { - return v ? v->size() : 0; -} - -// This is used as a helper type for accessing arrays. -template class Array { - typedef - typename flatbuffers::integral_constant::value> - scalar_tag; - typedef - typename flatbuffers::conditional::type - IndirectHelperType; - - public: - typedef typename IndirectHelper::return_type return_type; - typedef VectorIterator const_iterator; - typedef VectorReverseIterator const_reverse_iterator; - - FLATBUFFERS_CONSTEXPR uint16_t size() const { return length; } - - return_type Get(uoffset_t i) const { - FLATBUFFERS_ASSERT(i < size()); - return IndirectHelper::Read(Data(), i); - } - - return_type operator[](uoffset_t i) const { return Get(i); } - - // If this is a Vector of enums, T will be its storage type, not the enum - // type. This function makes it convenient to retrieve value with enum - // type E. - template E GetEnum(uoffset_t i) const { - return static_cast(Get(i)); - } - - const_iterator begin() const { return const_iterator(Data(), 0); } - const_iterator end() const { return const_iterator(Data(), size()); } - - const_reverse_iterator rbegin() const { - return const_reverse_iterator(end()); - } - const_reverse_iterator rend() const { return const_reverse_iterator(end()); } - - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - const_reverse_iterator crbegin() const { return rbegin(); } - const_reverse_iterator crend() const { return rend(); } - - // Get a mutable pointer to elements inside this array. - // This method used to mutate arrays of structs followed by a @p Mutate - // operation. For primitive types use @p Mutate directly. - // @warning Assignments and reads to/from the dereferenced pointer are not - // automatically converted to the correct endianness. - typename flatbuffers::conditional::type - GetMutablePointer(uoffset_t i) const { - FLATBUFFERS_ASSERT(i < size()); - return const_cast(&data()[i]); - } - - // Change elements if you have a non-const pointer to this object. - void Mutate(uoffset_t i, const T &val) { MutateImpl(scalar_tag(), i, val); } - - // The raw data in little endian format. Use with care. - const uint8_t *Data() const { return data_; } - - uint8_t *Data() { return data_; } - - // Similarly, but typed, much like std::vector::data - const T *data() const { return reinterpret_cast(Data()); } - T *data() { return reinterpret_cast(Data()); } - - protected: - void MutateImpl(flatbuffers::integral_constant, uoffset_t i, - const T &val) { - FLATBUFFERS_ASSERT(i < size()); - WriteScalar(data() + i, val); - } - - void MutateImpl(flatbuffers::integral_constant, uoffset_t i, - const T &val) { - *(GetMutablePointer(i)) = val; - } - - // This class is only used to access pre-existing data. Don't ever - // try to construct these manually. - // 'constexpr' allows us to use 'size()' at compile time. - // @note Must not use 'FLATBUFFERS_CONSTEXPR' here, as const is not allowed on - // a constructor. -#if defined(__cpp_constexpr) - constexpr Array(); -#else - Array(); -#endif - - uint8_t data_[length * sizeof(T)]; - - private: - // This class is a pointer. Copying will therefore create an invalid object. - // Private and unimplemented copy constructor. - Array(const Array &); - Array &operator=(const Array &); -}; - -// Specialization for Array[struct] with access using Offset pointer. -// This specialization used by idl_gen_text.cpp. -template class Array, length> { - static_assert(flatbuffers::is_same::value, "unexpected type T"); - - public: - typedef const void *return_type; - - const uint8_t *Data() const { return data_; } - - // Make idl_gen_text.cpp::PrintContainer happy. - return_type operator[](uoffset_t) const { - FLATBUFFERS_ASSERT(false); - return nullptr; - } - - private: - // This class is only used to access pre-existing data. - Array(); - Array(const Array &); - Array &operator=(const Array &); - - uint8_t data_[1]; -}; - -// Lexicographically compare two strings (possibly containing nulls), and -// return true if the first is less than the second. -static inline bool StringLessThan(const char *a_data, uoffset_t a_size, - const char *b_data, uoffset_t b_size) { - const auto cmp = memcmp(a_data, b_data, (std::min)(a_size, b_size)); - return cmp == 0 ? a_size < b_size : cmp < 0; -} - -struct String : public Vector { - const char *c_str() const { return reinterpret_cast(Data()); } - std::string str() const { return std::string(c_str(), size()); } - - // clang-format off - #ifdef FLATBUFFERS_HAS_STRING_VIEW - flatbuffers::string_view string_view() const { - return flatbuffers::string_view(c_str(), size()); - } - #endif // FLATBUFFERS_HAS_STRING_VIEW - // clang-format on - - bool operator<(const String &o) const { - return StringLessThan(this->data(), this->size(), o.data(), o.size()); - } -}; - -// Convenience function to get std::string from a String returning an empty -// string on null pointer. -static inline std::string GetString(const String *str) { - return str ? str->str() : ""; -} - -// Convenience function to get char* from a String returning an empty string on -// null pointer. -static inline const char *GetCstring(const String *str) { - return str ? str->c_str() : ""; -} - -// Allocator interface. This is flatbuffers-specific and meant only for -// `vector_downward` usage. -class Allocator { - public: - virtual ~Allocator() {} - - // Allocate `size` bytes of memory. - virtual uint8_t *allocate(size_t size) = 0; - - // Deallocate `size` bytes of memory at `p` allocated by this allocator. - virtual void deallocate(uint8_t *p, size_t size) = 0; - - // Reallocate `new_size` bytes of memory, replacing the old region of size - // `old_size` at `p`. In contrast to a normal realloc, this grows downwards, - // and is intended specifcally for `vector_downward` use. - // `in_use_back` and `in_use_front` indicate how much of `old_size` is - // actually in use at each end, and needs to be copied. - virtual uint8_t *reallocate_downward(uint8_t *old_p, size_t old_size, - size_t new_size, size_t in_use_back, - size_t in_use_front) { - FLATBUFFERS_ASSERT(new_size > old_size); // vector_downward only grows - uint8_t *new_p = allocate(new_size); - memcpy_downward(old_p, old_size, new_p, new_size, in_use_back, - in_use_front); - deallocate(old_p, old_size); - return new_p; - } - - protected: - // Called by `reallocate_downward` to copy memory from `old_p` of `old_size` - // to `new_p` of `new_size`. Only memory of size `in_use_front` and - // `in_use_back` will be copied from the front and back of the old memory - // allocation. - void memcpy_downward(uint8_t *old_p, size_t old_size, uint8_t *new_p, - size_t new_size, size_t in_use_back, - size_t in_use_front) { - memcpy(new_p + new_size - in_use_back, old_p + old_size - in_use_back, - in_use_back); - memcpy(new_p, old_p, in_use_front); - } -}; - -// DefaultAllocator uses new/delete to allocate memory regions -class DefaultAllocator : public Allocator { - public: - uint8_t *allocate(size_t size) FLATBUFFERS_OVERRIDE { - return new uint8_t[size]; - } - - void deallocate(uint8_t *p, size_t) FLATBUFFERS_OVERRIDE { delete[] p; } - - static void dealloc(void *p, size_t) { delete[] static_cast(p); } -}; - -// These functions allow for a null allocator to mean use the default allocator, -// as used by DetachedBuffer and vector_downward below. -// This is to avoid having a statically or dynamically allocated default -// allocator, or having to move it between the classes that may own it. -inline uint8_t *Allocate(Allocator *allocator, size_t size) { - return allocator ? allocator->allocate(size) - : DefaultAllocator().allocate(size); -} - -inline void Deallocate(Allocator *allocator, uint8_t *p, size_t size) { - if (allocator) - allocator->deallocate(p, size); - else - DefaultAllocator().deallocate(p, size); -} - -inline uint8_t *ReallocateDownward(Allocator *allocator, uint8_t *old_p, - size_t old_size, size_t new_size, - size_t in_use_back, size_t in_use_front) { - return allocator ? allocator->reallocate_downward(old_p, old_size, new_size, - in_use_back, in_use_front) - : DefaultAllocator().reallocate_downward( - old_p, old_size, new_size, in_use_back, in_use_front); -} - -// DetachedBuffer is a finished flatbuffer memory region, detached from its -// builder. The original memory region and allocator are also stored so that -// the DetachedBuffer can manage the memory lifetime. -class DetachedBuffer { - public: - DetachedBuffer() - : allocator_(nullptr), - own_allocator_(false), - buf_(nullptr), - reserved_(0), - cur_(nullptr), - size_(0) {} - - DetachedBuffer(Allocator *allocator, bool own_allocator, uint8_t *buf, - size_t reserved, uint8_t *cur, size_t sz) - : allocator_(allocator), - own_allocator_(own_allocator), - buf_(buf), - reserved_(reserved), - cur_(cur), - size_(sz) {} - - // clang-format off - #if !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - DetachedBuffer(DetachedBuffer &&other) - : allocator_(other.allocator_), - own_allocator_(other.own_allocator_), - buf_(other.buf_), - reserved_(other.reserved_), - cur_(other.cur_), - size_(other.size_) { - other.reset(); - } - // clang-format off - #endif // !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - - // clang-format off - #if !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - DetachedBuffer &operator=(DetachedBuffer &&other) { - if (this == &other) return *this; - - destroy(); - - allocator_ = other.allocator_; - own_allocator_ = other.own_allocator_; - buf_ = other.buf_; - reserved_ = other.reserved_; - cur_ = other.cur_; - size_ = other.size_; - - other.reset(); - - return *this; - } - // clang-format off - #endif // !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - - ~DetachedBuffer() { destroy(); } - - const uint8_t *data() const { return cur_; } - - uint8_t *data() { return cur_; } - - size_t size() const { return size_; } - - // clang-format off - #if 0 // disabled for now due to the ordering of classes in this header - template - bool Verify() const { - Verifier verifier(data(), size()); - return verifier.Verify(nullptr); - } - - template - const T* GetRoot() const { - return flatbuffers::GetRoot(data()); - } - - template - T* GetRoot() { - return flatbuffers::GetRoot(data()); - } - #endif - // clang-format on - - // clang-format off - #if !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - // These may change access mode, leave these at end of public section - FLATBUFFERS_DELETE_FUNC(DetachedBuffer(const DetachedBuffer &other)) - FLATBUFFERS_DELETE_FUNC( - DetachedBuffer &operator=(const DetachedBuffer &other)) - // clang-format off - #endif // !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - - protected: - Allocator *allocator_; - bool own_allocator_; - uint8_t *buf_; - size_t reserved_; - uint8_t *cur_; - size_t size_; - - inline void destroy() { - if (buf_) Deallocate(allocator_, buf_, reserved_); - if (own_allocator_ && allocator_) { delete allocator_; } - reset(); - } - - inline void reset() { - allocator_ = nullptr; - own_allocator_ = false; - buf_ = nullptr; - reserved_ = 0; - cur_ = nullptr; - size_ = 0; - } -}; - -// This is a minimal replication of std::vector functionality, -// except growing from higher to lower addresses. i.e push_back() inserts data -// in the lowest address in the vector. -// Since this vector leaves the lower part unused, we support a "scratch-pad" -// that can be stored there for temporary data, to share the allocated space. -// Essentially, this supports 2 std::vectors in a single buffer. -class vector_downward { - public: - explicit vector_downward(size_t initial_size, Allocator *allocator, - bool own_allocator, size_t buffer_minalign) - : allocator_(allocator), - own_allocator_(own_allocator), - initial_size_(initial_size), - buffer_minalign_(buffer_minalign), - reserved_(0), - buf_(nullptr), - cur_(nullptr), - scratch_(nullptr) {} - - // clang-format off - #if !defined(FLATBUFFERS_CPP98_STL) - vector_downward(vector_downward &&other) - #else - vector_downward(vector_downward &other) - #endif // defined(FLATBUFFERS_CPP98_STL) - // clang-format on - : allocator_(other.allocator_), - own_allocator_(other.own_allocator_), - initial_size_(other.initial_size_), - buffer_minalign_(other.buffer_minalign_), - reserved_(other.reserved_), - buf_(other.buf_), - cur_(other.cur_), - scratch_(other.scratch_) { - // No change in other.allocator_ - // No change in other.initial_size_ - // No change in other.buffer_minalign_ - other.own_allocator_ = false; - other.reserved_ = 0; - other.buf_ = nullptr; - other.cur_ = nullptr; - other.scratch_ = nullptr; - } - - // clang-format off - #if !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - vector_downward &operator=(vector_downward &&other) { - // Move construct a temporary and swap idiom - vector_downward temp(std::move(other)); - swap(temp); - return *this; - } - // clang-format off - #endif // defined(FLATBUFFERS_CPP98_STL) - // clang-format on - - ~vector_downward() { - clear_buffer(); - clear_allocator(); - } - - void reset() { - clear_buffer(); - clear(); - } - - void clear() { - if (buf_) { - cur_ = buf_ + reserved_; - } else { - reserved_ = 0; - cur_ = nullptr; - } - clear_scratch(); - } - - void clear_scratch() { scratch_ = buf_; } - - void clear_allocator() { - if (own_allocator_ && allocator_) { delete allocator_; } - allocator_ = nullptr; - own_allocator_ = false; - } - - void clear_buffer() { - if (buf_) Deallocate(allocator_, buf_, reserved_); - buf_ = nullptr; - } - - // Relinquish the pointer to the caller. - uint8_t *release_raw(size_t &allocated_bytes, size_t &offset) { - auto *buf = buf_; - allocated_bytes = reserved_; - offset = static_cast(cur_ - buf_); - - // release_raw only relinquishes the buffer ownership. - // Does not deallocate or reset the allocator. Destructor will do that. - buf_ = nullptr; - clear(); - return buf; - } - - // Relinquish the pointer to the caller. - DetachedBuffer release() { - // allocator ownership (if any) is transferred to DetachedBuffer. - DetachedBuffer fb(allocator_, own_allocator_, buf_, reserved_, cur_, - size()); - if (own_allocator_) { - allocator_ = nullptr; - own_allocator_ = false; - } - buf_ = nullptr; - clear(); - return fb; - } - - size_t ensure_space(size_t len) { - FLATBUFFERS_ASSERT(cur_ >= scratch_ && scratch_ >= buf_); - if (len > static_cast(cur_ - scratch_)) { reallocate(len); } - // Beyond this, signed offsets may not have enough range: - // (FlatBuffers > 2GB not supported). - FLATBUFFERS_ASSERT(size() < FLATBUFFERS_MAX_BUFFER_SIZE); - return len; - } - - inline uint8_t *make_space(size_t len) { - size_t space = ensure_space(len); - cur_ -= space; - return cur_; - } - - // Returns nullptr if using the DefaultAllocator. - Allocator *get_custom_allocator() { return allocator_; } - - uoffset_t size() const { - return static_cast(reserved_ - (cur_ - buf_)); - } - - uoffset_t scratch_size() const { - return static_cast(scratch_ - buf_); - } - - size_t capacity() const { return reserved_; } - - uint8_t *data() const { - FLATBUFFERS_ASSERT(cur_); - return cur_; - } - - uint8_t *scratch_data() const { - FLATBUFFERS_ASSERT(buf_); - return buf_; - } - - uint8_t *scratch_end() const { - FLATBUFFERS_ASSERT(scratch_); - return scratch_; - } - - uint8_t *data_at(size_t offset) const { return buf_ + reserved_ - offset; } - - void push(const uint8_t *bytes, size_t num) { - if (num > 0) { memcpy(make_space(num), bytes, num); } - } - - // Specialized version of push() that avoids memcpy call for small data. - template void push_small(const T &little_endian_t) { - make_space(sizeof(T)); - *reinterpret_cast(cur_) = little_endian_t; - } - - template void scratch_push_small(const T &t) { - ensure_space(sizeof(T)); - *reinterpret_cast(scratch_) = t; - scratch_ += sizeof(T); - } - - // fill() is most frequently called with small byte counts (<= 4), - // which is why we're using loops rather than calling memset. - void fill(size_t zero_pad_bytes) { - make_space(zero_pad_bytes); - for (size_t i = 0; i < zero_pad_bytes; i++) cur_[i] = 0; - } - - // Version for when we know the size is larger. - // Precondition: zero_pad_bytes > 0 - void fill_big(size_t zero_pad_bytes) { - memset(make_space(zero_pad_bytes), 0, zero_pad_bytes); - } - - void pop(size_t bytes_to_remove) { cur_ += bytes_to_remove; } - void scratch_pop(size_t bytes_to_remove) { scratch_ -= bytes_to_remove; } - - void swap(vector_downward &other) { - using std::swap; - swap(allocator_, other.allocator_); - swap(own_allocator_, other.own_allocator_); - swap(initial_size_, other.initial_size_); - swap(buffer_minalign_, other.buffer_minalign_); - swap(reserved_, other.reserved_); - swap(buf_, other.buf_); - swap(cur_, other.cur_); - swap(scratch_, other.scratch_); - } - - void swap_allocator(vector_downward &other) { - using std::swap; - swap(allocator_, other.allocator_); - swap(own_allocator_, other.own_allocator_); - } - - private: - // You shouldn't really be copying instances of this class. - FLATBUFFERS_DELETE_FUNC(vector_downward(const vector_downward &)) - FLATBUFFERS_DELETE_FUNC(vector_downward &operator=(const vector_downward &)) - - Allocator *allocator_; - bool own_allocator_; - size_t initial_size_; - size_t buffer_minalign_; - size_t reserved_; - uint8_t *buf_; - uint8_t *cur_; // Points at location between empty (below) and used (above). - uint8_t *scratch_; // Points to the end of the scratchpad in use. - - void reallocate(size_t len) { - auto old_reserved = reserved_; - auto old_size = size(); - auto old_scratch_size = scratch_size(); - reserved_ += - (std::max)(len, old_reserved ? old_reserved / 2 : initial_size_); - reserved_ = (reserved_ + buffer_minalign_ - 1) & ~(buffer_minalign_ - 1); - if (buf_) { - buf_ = ReallocateDownward(allocator_, buf_, old_reserved, reserved_, - old_size, old_scratch_size); - } else { - buf_ = Allocate(allocator_, reserved_); - } - cur_ = buf_ + reserved_ - old_size; - scratch_ = buf_ + old_scratch_size; - } -}; - -// Converts a Field ID to a virtual table offset. -inline voffset_t FieldIndexToOffset(voffset_t field_id) { - // Should correspond to what EndTable() below builds up. - const int fixed_fields = 2; // Vtable size and Object Size. - return static_cast((field_id + fixed_fields) * sizeof(voffset_t)); -} - -template -const T *data(const std::vector &v) { - // Eventually the returned pointer gets passed down to memcpy, so - // we need it to be non-null to avoid undefined behavior. - static uint8_t t; - return v.empty() ? reinterpret_cast(&t) : &v.front(); -} -template T *data(std::vector &v) { - // Eventually the returned pointer gets passed down to memcpy, so - // we need it to be non-null to avoid undefined behavior. - static uint8_t t; - return v.empty() ? reinterpret_cast(&t) : &v.front(); -} - -/// @endcond - -/// @addtogroup flatbuffers_cpp_api -/// @{ -/// @class FlatBufferBuilder -/// @brief Helper class to hold data needed in creation of a FlatBuffer. -/// To serialize data, you typically call one of the `Create*()` functions in -/// the generated code, which in turn call a sequence of `StartTable`/ -/// `PushElement`/`AddElement`/`EndTable`, or the builtin `CreateString`/ -/// `CreateVector` functions. Do this is depth-first order to build up a tree to -/// the root. `Finish()` wraps up the buffer ready for transport. -class FlatBufferBuilder { - public: - /// @brief Default constructor for FlatBufferBuilder. - /// @param[in] initial_size The initial size of the buffer, in bytes. Defaults - /// to `1024`. - /// @param[in] allocator An `Allocator` to use. If null will use - /// `DefaultAllocator`. - /// @param[in] own_allocator Whether the builder/vector should own the - /// allocator. Defaults to / `false`. - /// @param[in] buffer_minalign Force the buffer to be aligned to the given - /// minimum alignment upon reallocation. Only needed if you intend to store - /// types with custom alignment AND you wish to read the buffer in-place - /// directly after creation. - explicit FlatBufferBuilder( - size_t initial_size = 1024, Allocator *allocator = nullptr, - bool own_allocator = false, - size_t buffer_minalign = AlignOf()) - : buf_(initial_size, allocator, own_allocator, buffer_minalign), - num_field_loc(0), - max_voffset_(0), - nested(false), - finished(false), - minalign_(1), - force_defaults_(false), - dedup_vtables_(true), - string_pool(nullptr) { - EndianCheck(); - } - - // clang-format off - /// @brief Move constructor for FlatBufferBuilder. - #if !defined(FLATBUFFERS_CPP98_STL) - FlatBufferBuilder(FlatBufferBuilder &&other) - #else - FlatBufferBuilder(FlatBufferBuilder &other) - #endif // #if !defined(FLATBUFFERS_CPP98_STL) - : buf_(1024, nullptr, false, AlignOf()), - num_field_loc(0), - max_voffset_(0), - nested(false), - finished(false), - minalign_(1), - force_defaults_(false), - dedup_vtables_(true), - string_pool(nullptr) { - EndianCheck(); - // Default construct and swap idiom. - // Lack of delegating constructors in vs2010 makes it more verbose than needed. - Swap(other); - } - // clang-format on - - // clang-format off - #if !defined(FLATBUFFERS_CPP98_STL) - // clang-format on - /// @brief Move assignment operator for FlatBufferBuilder. - FlatBufferBuilder &operator=(FlatBufferBuilder &&other) { - // Move construct a temporary and swap idiom - FlatBufferBuilder temp(std::move(other)); - Swap(temp); - return *this; - } - // clang-format off - #endif // defined(FLATBUFFERS_CPP98_STL) - // clang-format on - - void Swap(FlatBufferBuilder &other) { - using std::swap; - buf_.swap(other.buf_); - swap(num_field_loc, other.num_field_loc); - swap(max_voffset_, other.max_voffset_); - swap(nested, other.nested); - swap(finished, other.finished); - swap(minalign_, other.minalign_); - swap(force_defaults_, other.force_defaults_); - swap(dedup_vtables_, other.dedup_vtables_); - swap(string_pool, other.string_pool); - } - - ~FlatBufferBuilder() { - if (string_pool) delete string_pool; - } - - void Reset() { - Clear(); // clear builder state - buf_.reset(); // deallocate buffer - } - - /// @brief Reset all the state in this FlatBufferBuilder so it can be reused - /// to construct another buffer. - void Clear() { - ClearOffsets(); - buf_.clear(); - nested = false; - finished = false; - minalign_ = 1; - if (string_pool) string_pool->clear(); - } - - /// @brief The current size of the serialized buffer, counting from the end. - /// @return Returns an `uoffset_t` with the current size of the buffer. - uoffset_t GetSize() const { return buf_.size(); } - - /// @brief Get the serialized buffer (after you call `Finish()`). - /// @return Returns an `uint8_t` pointer to the FlatBuffer data inside the - /// buffer. - uint8_t *GetBufferPointer() const { - Finished(); - return buf_.data(); - } - - /// @brief Get a pointer to an unfinished buffer. - /// @return Returns a `uint8_t` pointer to the unfinished buffer. - uint8_t *GetCurrentBufferPointer() const { return buf_.data(); } - - /// @brief Get the released pointer to the serialized buffer. - /// @warning Do NOT attempt to use this FlatBufferBuilder afterwards! - /// @return A `FlatBuffer` that owns the buffer and its allocator and - /// behaves similar to a `unique_ptr` with a deleter. - FLATBUFFERS_ATTRIBUTE(deprecated("use Release() instead")) - DetachedBuffer ReleaseBufferPointer() { - Finished(); - return buf_.release(); - } - - /// @brief Get the released DetachedBuffer. - /// @return A `DetachedBuffer` that owns the buffer and its allocator. - DetachedBuffer Release() { - Finished(); - return buf_.release(); - } - - /// @brief Get the released pointer to the serialized buffer. - /// @param size The size of the memory block containing - /// the serialized `FlatBuffer`. - /// @param offset The offset from the released pointer where the finished - /// `FlatBuffer` starts. - /// @return A raw pointer to the start of the memory block containing - /// the serialized `FlatBuffer`. - /// @remark If the allocator is owned, it gets deleted when the destructor is - /// called.. - uint8_t *ReleaseRaw(size_t &size, size_t &offset) { - Finished(); - return buf_.release_raw(size, offset); - } - - /// @brief get the minimum alignment this buffer needs to be accessed - /// properly. This is only known once all elements have been written (after - /// you call Finish()). You can use this information if you need to embed - /// a FlatBuffer in some other buffer, such that you can later read it - /// without first having to copy it into its own buffer. - size_t GetBufferMinAlignment() { - Finished(); - return minalign_; - } - - /// @cond FLATBUFFERS_INTERNAL - void Finished() const { - // If you get this assert, you're attempting to get access a buffer - // which hasn't been finished yet. Be sure to call - // FlatBufferBuilder::Finish with your root table. - // If you really need to access an unfinished buffer, call - // GetCurrentBufferPointer instead. - FLATBUFFERS_ASSERT(finished); - } - /// @endcond - - /// @brief In order to save space, fields that are set to their default value - /// don't get serialized into the buffer. - /// @param[in] fd When set to `true`, always serializes default values that - /// are set. Optional fields which are not set explicitly, will still not be - /// serialized. - void ForceDefaults(bool fd) { force_defaults_ = fd; } - - /// @brief By default vtables are deduped in order to save space. - /// @param[in] dedup When set to `true`, dedup vtables. - void DedupVtables(bool dedup) { dedup_vtables_ = dedup; } - - /// @cond FLATBUFFERS_INTERNAL - void Pad(size_t num_bytes) { buf_.fill(num_bytes); } - - void TrackMinAlign(size_t elem_size) { - if (elem_size > minalign_) minalign_ = elem_size; - } - - void Align(size_t elem_size) { - TrackMinAlign(elem_size); - buf_.fill(PaddingBytes(buf_.size(), elem_size)); - } - - void PushFlatBuffer(const uint8_t *bytes, size_t size) { - PushBytes(bytes, size); - finished = true; - } - - void PushBytes(const uint8_t *bytes, size_t size) { buf_.push(bytes, size); } - - void PopBytes(size_t amount) { buf_.pop(amount); } - - template void AssertScalarT() { - // The code assumes power of 2 sizes and endian-swap-ability. - static_assert(flatbuffers::is_scalar::value, "T must be a scalar type"); - } - - // Write a single aligned scalar to the buffer - template uoffset_t PushElement(T element) { - AssertScalarT(); - T litle_endian_element = EndianScalar(element); - Align(sizeof(T)); - buf_.push_small(litle_endian_element); - return GetSize(); - } - - template uoffset_t PushElement(Offset off) { - // Special case for offsets: see ReferTo below. - return PushElement(ReferTo(off.o)); - } - - // When writing fields, we track where they are, so we can create correct - // vtables later. - void TrackField(voffset_t field, uoffset_t off) { - FieldLoc fl = { off, field }; - buf_.scratch_push_small(fl); - num_field_loc++; - max_voffset_ = (std::max)(max_voffset_, field); - } - - // Like PushElement, but additionally tracks the field this represents. - template void AddElement(voffset_t field, T e, T def) { - // We don't serialize values equal to the default. - if (IsTheSameAs(e, def) && !force_defaults_) return; - auto off = PushElement(e); - TrackField(field, off); - } - - template void AddOffset(voffset_t field, Offset off) { - if (off.IsNull()) return; // Don't store. - AddElement(field, ReferTo(off.o), static_cast(0)); - } - - template void AddStruct(voffset_t field, const T *structptr) { - if (!structptr) return; // Default, don't store. - Align(AlignOf()); - buf_.push_small(*structptr); - TrackField(field, GetSize()); - } - - void AddStructOffset(voffset_t field, uoffset_t off) { - TrackField(field, off); - } - - // Offsets initially are relative to the end of the buffer (downwards). - // This function converts them to be relative to the current location - // in the buffer (when stored here), pointing upwards. - uoffset_t ReferTo(uoffset_t off) { - // Align to ensure GetSize() below is correct. - Align(sizeof(uoffset_t)); - // Offset must refer to something already in buffer. - FLATBUFFERS_ASSERT(off && off <= GetSize()); - return GetSize() - off + static_cast(sizeof(uoffset_t)); - } - - void NotNested() { - // If you hit this, you're trying to construct a Table/Vector/String - // during the construction of its parent table (between the MyTableBuilder - // and table.Finish(). - // Move the creation of these sub-objects to above the MyTableBuilder to - // not get this assert. - // Ignoring this assert may appear to work in simple cases, but the reason - // it is here is that storing objects in-line may cause vtable offsets - // to not fit anymore. It also leads to vtable duplication. - FLATBUFFERS_ASSERT(!nested); - // If you hit this, fields were added outside the scope of a table. - FLATBUFFERS_ASSERT(!num_field_loc); - } - - // From generated code (or from the parser), we call StartTable/EndTable - // with a sequence of AddElement calls in between. - uoffset_t StartTable() { - NotNested(); - nested = true; - return GetSize(); - } - - // This finishes one serialized object by generating the vtable if it's a - // table, comparing it against existing vtables, and writing the - // resulting vtable offset. - uoffset_t EndTable(uoffset_t start) { - // If you get this assert, a corresponding StartTable wasn't called. - FLATBUFFERS_ASSERT(nested); - // Write the vtable offset, which is the start of any Table. - // We fill it's value later. - auto vtableoffsetloc = PushElement(0); - // Write a vtable, which consists entirely of voffset_t elements. - // It starts with the number of offsets, followed by a type id, followed - // by the offsets themselves. In reverse: - // Include space for the last offset and ensure empty tables have a - // minimum size. - max_voffset_ = - (std::max)(static_cast(max_voffset_ + sizeof(voffset_t)), - FieldIndexToOffset(0)); - buf_.fill_big(max_voffset_); - auto table_object_size = vtableoffsetloc - start; - // Vtable use 16bit offsets. - FLATBUFFERS_ASSERT(table_object_size < 0x10000); - WriteScalar(buf_.data() + sizeof(voffset_t), - static_cast(table_object_size)); - WriteScalar(buf_.data(), max_voffset_); - // Write the offsets into the table - for (auto it = buf_.scratch_end() - num_field_loc * sizeof(FieldLoc); - it < buf_.scratch_end(); it += sizeof(FieldLoc)) { - auto field_location = reinterpret_cast(it); - auto pos = static_cast(vtableoffsetloc - field_location->off); - // If this asserts, it means you've set a field twice. - FLATBUFFERS_ASSERT( - !ReadScalar(buf_.data() + field_location->id)); - WriteScalar(buf_.data() + field_location->id, pos); - } - ClearOffsets(); - auto vt1 = reinterpret_cast(buf_.data()); - auto vt1_size = ReadScalar(vt1); - auto vt_use = GetSize(); - // See if we already have generated a vtable with this exact same - // layout before. If so, make it point to the old one, remove this one. - if (dedup_vtables_) { - for (auto it = buf_.scratch_data(); it < buf_.scratch_end(); - it += sizeof(uoffset_t)) { - auto vt_offset_ptr = reinterpret_cast(it); - auto vt2 = reinterpret_cast(buf_.data_at(*vt_offset_ptr)); - auto vt2_size = ReadScalar(vt2); - if (vt1_size != vt2_size || 0 != memcmp(vt2, vt1, vt1_size)) continue; - vt_use = *vt_offset_ptr; - buf_.pop(GetSize() - vtableoffsetloc); - break; - } - } - // If this is a new vtable, remember it. - if (vt_use == GetSize()) { buf_.scratch_push_small(vt_use); } - // Fill the vtable offset we created above. - // The offset points from the beginning of the object to where the - // vtable is stored. - // Offsets default direction is downward in memory for future format - // flexibility (storing all vtables at the start of the file). - WriteScalar(buf_.data_at(vtableoffsetloc), - static_cast(vt_use) - - static_cast(vtableoffsetloc)); - - nested = false; - return vtableoffsetloc; - } - - FLATBUFFERS_ATTRIBUTE(deprecated("call the version above instead")) - uoffset_t EndTable(uoffset_t start, voffset_t /*numfields*/) { - return EndTable(start); - } - - // This checks a required field has been set in a given table that has - // just been constructed. - template void Required(Offset table, voffset_t field); - - uoffset_t StartStruct(size_t alignment) { - Align(alignment); - return GetSize(); - } - - uoffset_t EndStruct() { return GetSize(); } - - void ClearOffsets() { - buf_.scratch_pop(num_field_loc * sizeof(FieldLoc)); - num_field_loc = 0; - max_voffset_ = 0; - } - - // Aligns such that when "len" bytes are written, an object can be written - // after it with "alignment" without padding. - void PreAlign(size_t len, size_t alignment) { - TrackMinAlign(alignment); - buf_.fill(PaddingBytes(GetSize() + len, alignment)); - } - template void PreAlign(size_t len) { - AssertScalarT(); - PreAlign(len, sizeof(T)); - } - /// @endcond - - /// @brief Store a string in the buffer, which can contain any binary data. - /// @param[in] str A const char pointer to the data to be stored as a string. - /// @param[in] len The number of bytes that should be stored from `str`. - /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(const char *str, size_t len) { - NotNested(); - PreAlign(len + 1); // Always 0-terminated. - buf_.fill(1); - PushBytes(reinterpret_cast(str), len); - PushElement(static_cast(len)); - return Offset(GetSize()); - } - - /// @brief Store a string in the buffer, which is null-terminated. - /// @param[in] str A const char pointer to a C-string to add to the buffer. - /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(const char *str) { - return CreateString(str, strlen(str)); - } - - /// @brief Store a string in the buffer, which is null-terminated. - /// @param[in] str A char pointer to a C-string to add to the buffer. - /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(char *str) { - return CreateString(str, strlen(str)); - } - - /// @brief Store a string in the buffer, which can contain any binary data. - /// @param[in] str A const reference to a std::string to store in the buffer. - /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(const std::string &str) { - return CreateString(str.c_str(), str.length()); - } - - // clang-format off - #ifdef FLATBUFFERS_HAS_STRING_VIEW - /// @brief Store a string in the buffer, which can contain any binary data. - /// @param[in] str A const string_view to copy in to the buffer. - /// @return Returns the offset in the buffer where the string starts. - Offset CreateString(flatbuffers::string_view str) { - return CreateString(str.data(), str.size()); - } - #endif // FLATBUFFERS_HAS_STRING_VIEW - // clang-format on - - /// @brief Store a string in the buffer, which can contain any binary data. - /// @param[in] str A const pointer to a `String` struct to add to the buffer. - /// @return Returns the offset in the buffer where the string starts - Offset CreateString(const String *str) { - return str ? CreateString(str->c_str(), str->size()) : 0; - } - - /// @brief Store a string in the buffer, which can contain any binary data. - /// @param[in] str A const reference to a std::string like type with support - /// of T::c_str() and T::length() to store in the buffer. - /// @return Returns the offset in the buffer where the string starts. - template Offset CreateString(const T &str) { - return CreateString(str.c_str(), str.length()); - } - - /// @brief Store a string in the buffer, which can contain any binary data. - /// If a string with this exact contents has already been serialized before, - /// instead simply returns the offset of the existing string. - /// @param[in] str A const char pointer to the data to be stored as a string. - /// @param[in] len The number of bytes that should be stored from `str`. - /// @return Returns the offset in the buffer where the string starts. - Offset CreateSharedString(const char *str, size_t len) { - if (!string_pool) - string_pool = new StringOffsetMap(StringOffsetCompare(buf_)); - auto size_before_string = buf_.size(); - // Must first serialize the string, since the set is all offsets into - // buffer. - auto off = CreateString(str, len); - auto it = string_pool->find(off); - // If it exists we reuse existing serialized data! - if (it != string_pool->end()) { - // We can remove the string we serialized. - buf_.pop(buf_.size() - size_before_string); - return *it; - } - // Record this string for future use. - string_pool->insert(off); - return off; - } - - /// @brief Store a string in the buffer, which null-terminated. - /// If a string with this exact contents has already been serialized before, - /// instead simply returns the offset of the existing string. - /// @param[in] str A const char pointer to a C-string to add to the buffer. - /// @return Returns the offset in the buffer where the string starts. - Offset CreateSharedString(const char *str) { - return CreateSharedString(str, strlen(str)); - } - - /// @brief Store a string in the buffer, which can contain any binary data. - /// If a string with this exact contents has already been serialized before, - /// instead simply returns the offset of the existing string. - /// @param[in] str A const reference to a std::string to store in the buffer. - /// @return Returns the offset in the buffer where the string starts. - Offset CreateSharedString(const std::string &str) { - return CreateSharedString(str.c_str(), str.length()); - } - - /// @brief Store a string in the buffer, which can contain any binary data. - /// If a string with this exact contents has already been serialized before, - /// instead simply returns the offset of the existing string. - /// @param[in] str A const pointer to a `String` struct to add to the buffer. - /// @return Returns the offset in the buffer where the string starts - Offset CreateSharedString(const String *str) { - return CreateSharedString(str->c_str(), str->size()); - } - - /// @cond FLATBUFFERS_INTERNAL - uoffset_t EndVector(size_t len) { - FLATBUFFERS_ASSERT(nested); // Hit if no corresponding StartVector. - nested = false; - return PushElement(static_cast(len)); - } - - void StartVector(size_t len, size_t elemsize) { - NotNested(); - nested = true; - PreAlign(len * elemsize); - PreAlign(len * elemsize, elemsize); // Just in case elemsize > uoffset_t. - } - - // Call this right before StartVector/CreateVector if you want to force the - // alignment to be something different than what the element size would - // normally dictate. - // This is useful when storing a nested_flatbuffer in a vector of bytes, - // or when storing SIMD floats, etc. - void ForceVectorAlignment(size_t len, size_t elemsize, size_t alignment) { - PreAlign(len * elemsize, alignment); - } - - // Similar to ForceVectorAlignment but for String fields. - void ForceStringAlignment(size_t len, size_t alignment) { - PreAlign((len + 1) * sizeof(char), alignment); - } - - /// @endcond - - /// @brief Serialize an array into a FlatBuffer `vector`. - /// @tparam T The data type of the array elements. - /// @param[in] v A pointer to the array of type `T` to serialize into the - /// buffer as a `vector`. - /// @param[in] len The number of elements to serialize. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template Offset> CreateVector(const T *v, size_t len) { - // If this assert hits, you're specifying a template argument that is - // causing the wrong overload to be selected, remove it. - AssertScalarT(); - StartVector(len, sizeof(T)); - // clang-format off - #if FLATBUFFERS_LITTLEENDIAN - PushBytes(reinterpret_cast(v), len * sizeof(T)); - #else - if (sizeof(T) == 1) { - PushBytes(reinterpret_cast(v), len); - } else { - for (auto i = len; i > 0; ) { - PushElement(v[--i]); - } - } - #endif - // clang-format on - return Offset>(EndVector(len)); - } - - template - Offset>> CreateVector(const Offset *v, size_t len) { - StartVector(len, sizeof(Offset)); - for (auto i = len; i > 0;) { PushElement(v[--i]); } - return Offset>>(EndVector(len)); - } - - /// @brief Serialize a `std::vector` into a FlatBuffer `vector`. - /// @tparam T The data type of the `std::vector` elements. - /// @param v A const reference to the `std::vector` to serialize into the - /// buffer as a `vector`. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template Offset> CreateVector(const std::vector &v) { - return CreateVector(data(v), v.size()); - } - - // vector may be implemented using a bit-set, so we can't access it as - // an array. Instead, read elements manually. - // Background: https://isocpp.org/blog/2012/11/on-vectorbool - Offset> CreateVector(const std::vector &v) { - StartVector(v.size(), sizeof(uint8_t)); - for (auto i = v.size(); i > 0;) { - PushElement(static_cast(v[--i])); - } - return Offset>(EndVector(v.size())); - } - - // clang-format off - #ifndef FLATBUFFERS_CPP98_STL - /// @brief Serialize values returned by a function into a FlatBuffer `vector`. - /// This is a convenience function that takes care of iteration for you. - /// @tparam T The data type of the `std::vector` elements. - /// @param f A function that takes the current iteration 0..vector_size-1 and - /// returns any type that you can construct a FlatBuffers vector out of. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template Offset> CreateVector(size_t vector_size, - const std::function &f) { - std::vector elems(vector_size); - for (size_t i = 0; i < vector_size; i++) elems[i] = f(i); - return CreateVector(elems); - } - #endif - // clang-format on - - /// @brief Serialize values returned by a function into a FlatBuffer `vector`. - /// This is a convenience function that takes care of iteration for you. - /// @tparam T The data type of the `std::vector` elements. - /// @param f A function that takes the current iteration 0..vector_size-1, - /// and the state parameter returning any type that you can construct a - /// FlatBuffers vector out of. - /// @param state State passed to f. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVector(size_t vector_size, F f, S *state) { - std::vector elems(vector_size); - for (size_t i = 0; i < vector_size; i++) elems[i] = f(i, state); - return CreateVector(elems); - } - - /// @brief Serialize a `std::vector` into a FlatBuffer `vector`. - /// This is a convenience function for a common case. - /// @param v A const reference to the `std::vector` to serialize into the - /// buffer as a `vector`. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - Offset>> CreateVectorOfStrings( - const std::vector &v) { - std::vector> offsets(v.size()); - for (size_t i = 0; i < v.size(); i++) offsets[i] = CreateString(v[i]); - return CreateVector(offsets); - } - - /// @brief Serialize an array of structs into a FlatBuffer `vector`. - /// @tparam T The data type of the struct array elements. - /// @param[in] v A pointer to the array of type `T` to serialize into the - /// buffer as a `vector`. - /// @param[in] len The number of elements to serialize. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVectorOfStructs(const T *v, size_t len) { - StartVector(len * sizeof(T) / AlignOf(), AlignOf()); - PushBytes(reinterpret_cast(v), sizeof(T) * len); - return Offset>(EndVector(len)); - } - - /// @brief Serialize an array of native structs into a FlatBuffer `vector`. - /// @tparam T The data type of the struct array elements. - /// @tparam S The data type of the native struct array elements. - /// @param[in] v A pointer to the array of type `S` to serialize into the - /// buffer as a `vector`. - /// @param[in] len The number of elements to serialize. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVectorOfNativeStructs(const S *v, - size_t len) { - extern T Pack(const S &); - std::vector vv(len); - std::transform(v, v + len, vv.begin(), Pack); - return CreateVectorOfStructs(data(vv), vv.size()); - } - - // clang-format off - #ifndef FLATBUFFERS_CPP98_STL - /// @brief Serialize an array of structs into a FlatBuffer `vector`. - /// @tparam T The data type of the struct array elements. - /// @param[in] filler A function that takes the current iteration 0..vector_size-1 - /// and a pointer to the struct that must be filled. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - /// This is mostly useful when flatbuffers are generated with mutation - /// accessors. - template Offset> CreateVectorOfStructs( - size_t vector_size, const std::function &filler) { - T* structs = StartVectorOfStructs(vector_size); - for (size_t i = 0; i < vector_size; i++) { - filler(i, structs); - structs++; - } - return EndVectorOfStructs(vector_size); - } - #endif - // clang-format on - - /// @brief Serialize an array of structs into a FlatBuffer `vector`. - /// @tparam T The data type of the struct array elements. - /// @param[in] f A function that takes the current iteration 0..vector_size-1, - /// a pointer to the struct that must be filled and the state argument. - /// @param[in] state Arbitrary state to pass to f. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - /// This is mostly useful when flatbuffers are generated with mutation - /// accessors. - template - Offset> CreateVectorOfStructs(size_t vector_size, F f, - S *state) { - T *structs = StartVectorOfStructs(vector_size); - for (size_t i = 0; i < vector_size; i++) { - f(i, structs, state); - structs++; - } - return EndVectorOfStructs(vector_size); - } - - /// @brief Serialize a `std::vector` of structs into a FlatBuffer `vector`. - /// @tparam T The data type of the `std::vector` struct elements. - /// @param[in] v A const reference to the `std::vector` of structs to - /// serialize into the buffer as a `vector`. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVectorOfStructs( - const std::vector &v) { - return CreateVectorOfStructs(data(v), v.size()); - } - - /// @brief Serialize a `std::vector` of native structs into a FlatBuffer - /// `vector`. - /// @tparam T The data type of the `std::vector` struct elements. - /// @tparam S The data type of the `std::vector` native struct elements. - /// @param[in] v A const reference to the `std::vector` of structs to - /// serialize into the buffer as a `vector`. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVectorOfNativeStructs( - const std::vector &v) { - return CreateVectorOfNativeStructs(data(v), v.size()); - } - - /// @cond FLATBUFFERS_INTERNAL - template struct StructKeyComparator { - bool operator()(const T &a, const T &b) const { - return a.KeyCompareLessThan(&b); - } - - private: - StructKeyComparator &operator=(const StructKeyComparator &); - }; - /// @endcond - - /// @brief Serialize a `std::vector` of structs into a FlatBuffer `vector` - /// in sorted order. - /// @tparam T The data type of the `std::vector` struct elements. - /// @param[in] v A const reference to the `std::vector` of structs to - /// serialize into the buffer as a `vector`. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVectorOfSortedStructs(std::vector *v) { - return CreateVectorOfSortedStructs(data(*v), v->size()); - } - - /// @brief Serialize a `std::vector` of native structs into a FlatBuffer - /// `vector` in sorted order. - /// @tparam T The data type of the `std::vector` struct elements. - /// @tparam S The data type of the `std::vector` native struct elements. - /// @param[in] v A const reference to the `std::vector` of structs to - /// serialize into the buffer as a `vector`. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVectorOfSortedNativeStructs( - std::vector *v) { - return CreateVectorOfSortedNativeStructs(data(*v), v->size()); - } - - /// @brief Serialize an array of structs into a FlatBuffer `vector` in sorted - /// order. - /// @tparam T The data type of the struct array elements. - /// @param[in] v A pointer to the array of type `T` to serialize into the - /// buffer as a `vector`. - /// @param[in] len The number of elements to serialize. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVectorOfSortedStructs(T *v, size_t len) { - std::sort(v, v + len, StructKeyComparator()); - return CreateVectorOfStructs(v, len); - } - - /// @brief Serialize an array of native structs into a FlatBuffer `vector` in - /// sorted order. - /// @tparam T The data type of the struct array elements. - /// @tparam S The data type of the native struct array elements. - /// @param[in] v A pointer to the array of type `S` to serialize into the - /// buffer as a `vector`. - /// @param[in] len The number of elements to serialize. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset> CreateVectorOfSortedNativeStructs(S *v, - size_t len) { - extern T Pack(const S &); - typedef T (*Pack_t)(const S &); - std::vector vv(len); - std::transform(v, v + len, vv.begin(), static_cast(Pack)); - return CreateVectorOfSortedStructs(vv, len); - } - - /// @cond FLATBUFFERS_INTERNAL - template struct TableKeyComparator { - TableKeyComparator(vector_downward &buf) : buf_(buf) {} - TableKeyComparator(const TableKeyComparator &other) : buf_(other.buf_) {} - bool operator()(const Offset &a, const Offset &b) const { - auto table_a = reinterpret_cast(buf_.data_at(a.o)); - auto table_b = reinterpret_cast(buf_.data_at(b.o)); - return table_a->KeyCompareLessThan(table_b); - } - vector_downward &buf_; - - private: - TableKeyComparator &operator=(const TableKeyComparator &other) { - buf_ = other.buf_; - return *this; - } - }; - /// @endcond - - /// @brief Serialize an array of `table` offsets as a `vector` in the buffer - /// in sorted order. - /// @tparam T The data type that the offset refers to. - /// @param[in] v An array of type `Offset` that contains the `table` - /// offsets to store in the buffer in sorted order. - /// @param[in] len The number of elements to store in the `vector`. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset>> CreateVectorOfSortedTables(Offset *v, - size_t len) { - std::sort(v, v + len, TableKeyComparator(buf_)); - return CreateVector(v, len); - } - - /// @brief Serialize an array of `table` offsets as a `vector` in the buffer - /// in sorted order. - /// @tparam T The data type that the offset refers to. - /// @param[in] v An array of type `Offset` that contains the `table` - /// offsets to store in the buffer in sorted order. - /// @return Returns a typed `Offset` into the serialized data indicating - /// where the vector is stored. - template - Offset>> CreateVectorOfSortedTables( - std::vector> *v) { - return CreateVectorOfSortedTables(data(*v), v->size()); - } - - /// @brief Specialized version of `CreateVector` for non-copying use cases. - /// Write the data any time later to the returned buffer pointer `buf`. - /// @param[in] len The number of elements to store in the `vector`. - /// @param[in] elemsize The size of each element in the `vector`. - /// @param[out] buf A pointer to a `uint8_t` pointer that can be - /// written to at a later time to serialize the data into a `vector` - /// in the buffer. - uoffset_t CreateUninitializedVector(size_t len, size_t elemsize, - uint8_t **buf) { - NotNested(); - StartVector(len, elemsize); - buf_.make_space(len * elemsize); - auto vec_start = GetSize(); - auto vec_end = EndVector(len); - *buf = buf_.data_at(vec_start); - return vec_end; - } - - /// @brief Specialized version of `CreateVector` for non-copying use cases. - /// Write the data any time later to the returned buffer pointer `buf`. - /// @tparam T The data type of the data that will be stored in the buffer - /// as a `vector`. - /// @param[in] len The number of elements to store in the `vector`. - /// @param[out] buf A pointer to a pointer of type `T` that can be - /// written to at a later time to serialize the data into a `vector` - /// in the buffer. - template - Offset> CreateUninitializedVector(size_t len, T **buf) { - AssertScalarT(); - return CreateUninitializedVector(len, sizeof(T), - reinterpret_cast(buf)); - } - - template - Offset> CreateUninitializedVectorOfStructs(size_t len, - T **buf) { - return CreateUninitializedVector(len, sizeof(T), - reinterpret_cast(buf)); - } - - // @brief Create a vector of scalar type T given as input a vector of scalar - // type U, useful with e.g. pre "enum class" enums, or any existing scalar - // data of the wrong type. - template - Offset> CreateVectorScalarCast(const U *v, size_t len) { - AssertScalarT(); - AssertScalarT(); - StartVector(len, sizeof(T)); - for (auto i = len; i > 0;) { PushElement(static_cast(v[--i])); } - return Offset>(EndVector(len)); - } - - /// @brief Write a struct by itself, typically to be part of a union. - template Offset CreateStruct(const T &structobj) { - NotNested(); - Align(AlignOf()); - buf_.push_small(structobj); - return Offset(GetSize()); - } - - /// @brief The length of a FlatBuffer file header. - static const size_t kFileIdentifierLength = 4; - - /// @brief Finish serializing a buffer by writing the root offset. - /// @param[in] file_identifier If a `file_identifier` is given, the buffer - /// will be prefixed with a standard FlatBuffers file header. - template - void Finish(Offset root, const char *file_identifier = nullptr) { - Finish(root.o, file_identifier, false); - } - - /// @brief Finish a buffer with a 32 bit size field pre-fixed (size of the - /// buffer following the size field). These buffers are NOT compatible - /// with standard buffers created by Finish, i.e. you can't call GetRoot - /// on them, you have to use GetSizePrefixedRoot instead. - /// All >32 bit quantities in this buffer will be aligned when the whole - /// size pre-fixed buffer is aligned. - /// These kinds of buffers are useful for creating a stream of FlatBuffers. - template - void FinishSizePrefixed(Offset root, - const char *file_identifier = nullptr) { - Finish(root.o, file_identifier, true); - } - - void SwapBufAllocator(FlatBufferBuilder &other) { - buf_.swap_allocator(other.buf_); - } - - protected: - // You shouldn't really be copying instances of this class. - FlatBufferBuilder(const FlatBufferBuilder &); - FlatBufferBuilder &operator=(const FlatBufferBuilder &); - - void Finish(uoffset_t root, const char *file_identifier, bool size_prefix) { - NotNested(); - buf_.clear_scratch(); - // This will cause the whole buffer to be aligned. - PreAlign((size_prefix ? sizeof(uoffset_t) : 0) + sizeof(uoffset_t) + - (file_identifier ? kFileIdentifierLength : 0), - minalign_); - if (file_identifier) { - FLATBUFFERS_ASSERT(strlen(file_identifier) == kFileIdentifierLength); - PushBytes(reinterpret_cast(file_identifier), - kFileIdentifierLength); - } - PushElement(ReferTo(root)); // Location of root. - if (size_prefix) { PushElement(GetSize()); } - finished = true; - } - - struct FieldLoc { - uoffset_t off; - voffset_t id; - }; - - vector_downward buf_; - - // Accumulating offsets of table members while it is being built. - // We store these in the scratch pad of buf_, after the vtable offsets. - uoffset_t num_field_loc; - // Track how much of the vtable is in use, so we can output the most compact - // possible vtable. - voffset_t max_voffset_; - - // Ensure objects are not nested. - bool nested; - - // Ensure the buffer is finished before it is being accessed. - bool finished; - - size_t minalign_; - - bool force_defaults_; // Serialize values equal to their defaults anyway. - - bool dedup_vtables_; - - struct StringOffsetCompare { - StringOffsetCompare(const vector_downward &buf) : buf_(&buf) {} - bool operator()(const Offset &a, const Offset &b) const { - auto stra = reinterpret_cast(buf_->data_at(a.o)); - auto strb = reinterpret_cast(buf_->data_at(b.o)); - return StringLessThan(stra->data(), stra->size(), strb->data(), - strb->size()); - } - const vector_downward *buf_; - }; - - // For use with CreateSharedString. Instantiated on first use only. - typedef std::set, StringOffsetCompare> StringOffsetMap; - StringOffsetMap *string_pool; - - private: - // Allocates space for a vector of structures. - // Must be completed with EndVectorOfStructs(). - template T *StartVectorOfStructs(size_t vector_size) { - StartVector(vector_size * sizeof(T) / AlignOf(), AlignOf()); - return reinterpret_cast(buf_.make_space(vector_size * sizeof(T))); - } - - // End the vector of structues in the flatbuffers. - // Vector should have previously be started with StartVectorOfStructs(). - template - Offset> EndVectorOfStructs(size_t vector_size) { - return Offset>(EndVector(vector_size)); - } -}; -/// @} - -/// @cond FLATBUFFERS_INTERNAL -// Helpers to get a typed pointer to the root object contained in the buffer. -template T *GetMutableRoot(void *buf) { - EndianCheck(); - return reinterpret_cast( - reinterpret_cast(buf) + - EndianScalar(*reinterpret_cast(buf))); -} - -template const T *GetRoot(const void *buf) { - return GetMutableRoot(const_cast(buf)); -} - -template const T *GetSizePrefixedRoot(const void *buf) { - return GetRoot(reinterpret_cast(buf) + sizeof(uoffset_t)); -} - -/// Helpers to get a typed pointer to objects that are currently being built. -/// @warning Creating new objects will lead to reallocations and invalidates -/// the pointer! -template -T *GetMutableTemporaryPointer(FlatBufferBuilder &fbb, Offset offset) { - return reinterpret_cast(fbb.GetCurrentBufferPointer() + fbb.GetSize() - - offset.o); -} - -template -const T *GetTemporaryPointer(FlatBufferBuilder &fbb, Offset offset) { - return GetMutableTemporaryPointer(fbb, offset); -} - -/// @brief Get a pointer to the the file_identifier section of the buffer. -/// @return Returns a const char pointer to the start of the file_identifier -/// characters in the buffer. The returned char * has length -/// 'flatbuffers::FlatBufferBuilder::kFileIdentifierLength'. -/// This function is UNDEFINED for FlatBuffers whose schema does not include -/// a file_identifier (likely points at padding or the start of a the root -/// vtable). -inline const char *GetBufferIdentifier(const void *buf, - bool size_prefixed = false) { - return reinterpret_cast(buf) + - ((size_prefixed) ? 2 * sizeof(uoffset_t) : sizeof(uoffset_t)); -} - -// Helper to see if the identifier in a buffer has the expected value. -inline bool BufferHasIdentifier(const void *buf, const char *identifier, - bool size_prefixed = false) { - return strncmp(GetBufferIdentifier(buf, size_prefixed), identifier, - FlatBufferBuilder::kFileIdentifierLength) == 0; -} - -// Helper class to verify the integrity of a FlatBuffer -class Verifier FLATBUFFERS_FINAL_CLASS { - public: - Verifier(const uint8_t *buf, size_t buf_len, uoffset_t _max_depth = 64, - uoffset_t _max_tables = 1000000, bool _check_alignment = true) - : buf_(buf), - size_(buf_len), - depth_(0), - max_depth_(_max_depth), - num_tables_(0), - max_tables_(_max_tables), - upper_bound_(0), - check_alignment_(_check_alignment) { - FLATBUFFERS_ASSERT(size_ < FLATBUFFERS_MAX_BUFFER_SIZE); - } - - // Central location where any verification failures register. - bool Check(bool ok) const { - // clang-format off - #ifdef FLATBUFFERS_DEBUG_VERIFICATION_FAILURE - FLATBUFFERS_ASSERT(ok); - #endif - #ifdef FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE - if (!ok) - upper_bound_ = 0; - #endif - // clang-format on - return ok; - } - - // Verify any range within the buffer. - bool Verify(size_t elem, size_t elem_len) const { - // clang-format off - #ifdef FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE - auto upper_bound = elem + elem_len; - if (upper_bound_ < upper_bound) - upper_bound_ = upper_bound; - #endif - // clang-format on - return Check(elem_len < size_ && elem <= size_ - elem_len); - } - - template bool VerifyAlignment(size_t elem) const { - return Check((elem & (sizeof(T) - 1)) == 0 || !check_alignment_); - } - - // Verify a range indicated by sizeof(T). - template bool Verify(size_t elem) const { - return VerifyAlignment(elem) && Verify(elem, sizeof(T)); - } - - bool VerifyFromPointer(const uint8_t *p, size_t len) { - auto o = static_cast(p - buf_); - return Verify(o, len); - } - - // Verify relative to a known-good base pointer. - bool Verify(const uint8_t *base, voffset_t elem_off, size_t elem_len) const { - return Verify(static_cast(base - buf_) + elem_off, elem_len); - } - - template - bool Verify(const uint8_t *base, voffset_t elem_off) const { - return Verify(static_cast(base - buf_) + elem_off, sizeof(T)); - } - - // Verify a pointer (may be NULL) of a table type. - template bool VerifyTable(const T *table) { - return !table || table->Verify(*this); - } - - // Verify a pointer (may be NULL) of any vector type. - template bool VerifyVector(const Vector *vec) const { - return !vec || VerifyVectorOrString(reinterpret_cast(vec), - sizeof(T)); - } - - // Verify a pointer (may be NULL) of a vector to struct. - template bool VerifyVector(const Vector *vec) const { - return VerifyVector(reinterpret_cast *>(vec)); - } - - // Verify a pointer (may be NULL) to string. - bool VerifyString(const String *str) const { - size_t end; - return !str || (VerifyVectorOrString(reinterpret_cast(str), - 1, &end) && - Verify(end, 1) && // Must have terminator - Check(buf_[end] == '\0')); // Terminating byte must be 0. - } - - // Common code between vectors and strings. - bool VerifyVectorOrString(const uint8_t *vec, size_t elem_size, - size_t *end = nullptr) const { - auto veco = static_cast(vec - buf_); - // Check we can read the size field. - if (!Verify(veco)) return false; - // Check the whole array. If this is a string, the byte past the array - // must be 0. - auto size = ReadScalar(vec); - auto max_elems = FLATBUFFERS_MAX_BUFFER_SIZE / elem_size; - if (!Check(size < max_elems)) - return false; // Protect against byte_size overflowing. - auto byte_size = sizeof(size) + elem_size * size; - if (end) *end = veco + byte_size; - return Verify(veco, byte_size); - } - - // Special case for string contents, after the above has been called. - bool VerifyVectorOfStrings(const Vector> *vec) const { - if (vec) { - for (uoffset_t i = 0; i < vec->size(); i++) { - if (!VerifyString(vec->Get(i))) return false; - } - } - return true; - } - - // Special case for table contents, after the above has been called. - template bool VerifyVectorOfTables(const Vector> *vec) { - if (vec) { - for (uoffset_t i = 0; i < vec->size(); i++) { - if (!vec->Get(i)->Verify(*this)) return false; - } - } - return true; - } - - __supress_ubsan__("unsigned-integer-overflow") bool VerifyTableStart( - const uint8_t *table) { - // Check the vtable offset. - auto tableo = static_cast(table - buf_); - if (!Verify(tableo)) return false; - // This offset may be signed, but doing the subtraction unsigned always - // gives the result we want. - auto vtableo = tableo - static_cast(ReadScalar(table)); - // Check the vtable size field, then check vtable fits in its entirety. - return VerifyComplexity() && Verify(vtableo) && - VerifyAlignment(ReadScalar(buf_ + vtableo)) && - Verify(vtableo, ReadScalar(buf_ + vtableo)); - } - - template - bool VerifyBufferFromStart(const char *identifier, size_t start) { - if (identifier && (size_ < 2 * sizeof(flatbuffers::uoffset_t) || - !BufferHasIdentifier(buf_ + start, identifier))) { - return false; - } - - // Call T::Verify, which must be in the generated code for this type. - auto o = VerifyOffset(start); - return o && reinterpret_cast(buf_ + start + o)->Verify(*this) - // clang-format off - #ifdef FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE - && GetComputedSize() - #endif - ; - // clang-format on - } - - // Verify this whole buffer, starting with root type T. - template bool VerifyBuffer() { return VerifyBuffer(nullptr); } - - template bool VerifyBuffer(const char *identifier) { - return VerifyBufferFromStart(identifier, 0); - } - - template bool VerifySizePrefixedBuffer(const char *identifier) { - return Verify(0U) && - ReadScalar(buf_) == size_ - sizeof(uoffset_t) && - VerifyBufferFromStart(identifier, sizeof(uoffset_t)); - } - - uoffset_t VerifyOffset(size_t start) const { - if (!Verify(start)) return 0; - auto o = ReadScalar(buf_ + start); - // May not point to itself. - if (!Check(o != 0)) return 0; - // Can't wrap around / buffers are max 2GB. - if (!Check(static_cast(o) >= 0)) return 0; - // Must be inside the buffer to create a pointer from it (pointer outside - // buffer is UB). - if (!Verify(start + o, 1)) return 0; - return o; - } - - uoffset_t VerifyOffset(const uint8_t *base, voffset_t start) const { - return VerifyOffset(static_cast(base - buf_) + start); - } - - // Called at the start of a table to increase counters measuring data - // structure depth and amount, and possibly bails out with false if - // limits set by the constructor have been hit. Needs to be balanced - // with EndTable(). - bool VerifyComplexity() { - depth_++; - num_tables_++; - return Check(depth_ <= max_depth_ && num_tables_ <= max_tables_); - } - - // Called at the end of a table to pop the depth count. - bool EndTable() { - depth_--; - return true; - } - - // Returns the message size in bytes - size_t GetComputedSize() const { - // clang-format off - #ifdef FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE - uintptr_t size = upper_bound_; - // Align the size to uoffset_t - size = (size - 1 + sizeof(uoffset_t)) & ~(sizeof(uoffset_t) - 1); - return (size > size_) ? 0 : size; - #else - // Must turn on FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE for this to work. - (void)upper_bound_; - FLATBUFFERS_ASSERT(false); - return 0; - #endif - // clang-format on - } - - private: - const uint8_t *buf_; - size_t size_; - uoffset_t depth_; - uoffset_t max_depth_; - uoffset_t num_tables_; - uoffset_t max_tables_; - mutable size_t upper_bound_; - bool check_alignment_; -}; - -// Convenient way to bundle a buffer and its length, to pass it around -// typed by its root. -// A BufferRef does not own its buffer. -struct BufferRefBase {}; // for std::is_base_of -template struct BufferRef : BufferRefBase { - BufferRef() : buf(nullptr), len(0), must_free(false) {} - BufferRef(uint8_t *_buf, uoffset_t _len) - : buf(_buf), len(_len), must_free(false) {} - - ~BufferRef() { - if (must_free) free(buf); - } - - const T *GetRoot() const { return flatbuffers::GetRoot(buf); } - - bool Verify() { - Verifier verifier(buf, len); - return verifier.VerifyBuffer(nullptr); - } - - uint8_t *buf; - uoffset_t len; - bool must_free; -}; - -// "structs" are flat structures that do not have an offset table, thus -// always have all members present and do not support forwards/backwards -// compatible extensions. - -class Struct FLATBUFFERS_FINAL_CLASS { - public: - template T GetField(uoffset_t o) const { - return ReadScalar(&data_[o]); - } - - template T GetStruct(uoffset_t o) const { - return reinterpret_cast(&data_[o]); - } - - const uint8_t *GetAddressOf(uoffset_t o) const { return &data_[o]; } - uint8_t *GetAddressOf(uoffset_t o) { return &data_[o]; } - - private: - // private constructor & copy constructor: you obtain instances of this - // class by pointing to existing data only - Struct(); - Struct(const Struct &); - Struct &operator=(const Struct &); - - uint8_t data_[1]; -}; - -// "tables" use an offset table (possibly shared) that allows fields to be -// omitted and added at will, but uses an extra indirection to read. -class Table { - public: - const uint8_t *GetVTable() const { - return data_ - ReadScalar(data_); - } - - // This gets the field offset for any of the functions below it, or 0 - // if the field was not present. - voffset_t GetOptionalFieldOffset(voffset_t field) const { - // The vtable offset is always at the start. - auto vtable = GetVTable(); - // The first element is the size of the vtable (fields + type id + itself). - auto vtsize = ReadScalar(vtable); - // If the field we're accessing is outside the vtable, we're reading older - // data, so it's the same as if the offset was 0 (not present). - return field < vtsize ? ReadScalar(vtable + field) : 0; - } - - template T GetField(voffset_t field, T defaultval) const { - auto field_offset = GetOptionalFieldOffset(field); - return field_offset ? ReadScalar(data_ + field_offset) : defaultval; - } - - template P GetPointer(voffset_t field) { - auto field_offset = GetOptionalFieldOffset(field); - auto p = data_ + field_offset; - return field_offset ? reinterpret_cast

(p + ReadScalar(p)) - : nullptr; - } - template P GetPointer(voffset_t field) const { - return const_cast

NameTypeSize (Bytes)Delete
" - "
" - "
NameTypeSize (Bytes)Delete
" + "
" + "
uri); + httpd_resp_sendstr_chunk(req, "/fileserver"); + httpd_resp_sendstr_chunk(req, uripath); httpd_resp_sendstr_chunk(req, entry->d_name); if (entry->d_type == DT_DIR) { httpd_resp_sendstr_chunk(req, "/"); @@ -184,11 +189,13 @@ static esp_err_t http_resp_dir_html(httpd_req_t *req, const char *dirpath) httpd_resp_sendstr_chunk(req, entrytype); httpd_resp_sendstr_chunk(req, ""); httpd_resp_sendstr_chunk(req, entrysize); - httpd_resp_sendstr_chunk(req, ""); - httpd_resp_sendstr_chunk(req, "
uri + strlen("/fileserver")); - httpd_resp_sendstr_chunk(req, entry->d_name); - httpd_resp_sendstr_chunk(req, "\">
"); + if (!readonly) { + httpd_resp_sendstr_chunk(req, "		"); + httpd_resp_sendstr_chunk(req, "
d_name); + httpd_resp_sendstr_chunk(req, "\">
"); + } httpd_resp_sendstr_chunk(req, "
(this)->GetPointer

(field); - } - - template P GetStruct(voffset_t field) const { - auto field_offset = GetOptionalFieldOffset(field); - auto p = const_cast(data_ + field_offset); - return field_offset ? reinterpret_cast

(p) : nullptr; - } - - template bool SetField(voffset_t field, T val, T def) { - auto field_offset = GetOptionalFieldOffset(field); - if (!field_offset) return IsTheSameAs(val, def); - WriteScalar(data_ + field_offset, val); - return true; - } - - bool SetPointer(voffset_t field, const uint8_t *val) { - auto field_offset = GetOptionalFieldOffset(field); - if (!field_offset) return false; - WriteScalar(data_ + field_offset, - static_cast(val - (data_ + field_offset))); - return true; - } - - uint8_t *GetAddressOf(voffset_t field) { - auto field_offset = GetOptionalFieldOffset(field); - return field_offset ? data_ + field_offset : nullptr; - } - const uint8_t *GetAddressOf(voffset_t field) const { - return const_cast

(this)->GetAddressOf(field); - } - - bool CheckField(voffset_t field) const { - return GetOptionalFieldOffset(field) != 0; - } - - // Verify the vtable of this table. - // Call this once per table, followed by VerifyField once per field. - bool VerifyTableStart(Verifier &verifier) const { - return verifier.VerifyTableStart(data_); - } - - // Verify a particular field. - template - bool VerifyField(const Verifier &verifier, voffset_t field) const { - // Calling GetOptionalFieldOffset should be safe now thanks to - // VerifyTable(). - auto field_offset = GetOptionalFieldOffset(field); - // Check the actual field. - return !field_offset || verifier.Verify(data_, field_offset); - } - - // VerifyField for required fields. - template - bool VerifyFieldRequired(const Verifier &verifier, voffset_t field) const { - auto field_offset = GetOptionalFieldOffset(field); - return verifier.Check(field_offset != 0) && - verifier.Verify(data_, field_offset); - } - - // Versions for offsets. - bool VerifyOffset(const Verifier &verifier, voffset_t field) const { - auto field_offset = GetOptionalFieldOffset(field); - return !field_offset || verifier.VerifyOffset(data_, field_offset); - } - - bool VerifyOffsetRequired(const Verifier &verifier, voffset_t field) const { - auto field_offset = GetOptionalFieldOffset(field); - return verifier.Check(field_offset != 0) && - verifier.VerifyOffset(data_, field_offset); - } - - private: - // private constructor & copy constructor: you obtain instances of this - // class by pointing to existing data only - Table(); - Table(const Table &other); - Table &operator=(const Table &); - - uint8_t data_[1]; -}; - -template -void FlatBufferBuilder::Required(Offset table, voffset_t field) { - auto table_ptr = reinterpret_cast(buf_.data_at(table.o)); - bool ok = table_ptr->GetOptionalFieldOffset(field) != 0; - // If this fails, the caller will show what field needs to be set. - FLATBUFFERS_ASSERT(ok); - (void)ok; -} - -/// @brief This can compute the start of a FlatBuffer from a root pointer, i.e. -/// it is the opposite transformation of GetRoot(). -/// This may be useful if you want to pass on a root and have the recipient -/// delete the buffer afterwards. -inline const uint8_t *GetBufferStartFromRootPointer(const void *root) { - auto table = reinterpret_cast(root); - auto vtable = table->GetVTable(); - // Either the vtable is before the root or after the root. - auto start = (std::min)(vtable, reinterpret_cast(root)); - // Align to at least sizeof(uoffset_t). - start = reinterpret_cast(reinterpret_cast(start) & - ~(sizeof(uoffset_t) - 1)); - // Additionally, there may be a file_identifier in the buffer, and the root - // offset. The buffer may have been aligned to any size between - // sizeof(uoffset_t) and FLATBUFFERS_MAX_ALIGNMENT (see "force_align"). - // Sadly, the exact alignment is only known when constructing the buffer, - // since it depends on the presence of values with said alignment properties. - // So instead, we simply look at the next uoffset_t values (root, - // file_identifier, and alignment padding) to see which points to the root. - // None of the other values can "impersonate" the root since they will either - // be 0 or four ASCII characters. - static_assert(FlatBufferBuilder::kFileIdentifierLength == sizeof(uoffset_t), - "file_identifier is assumed to be the same size as uoffset_t"); - for (auto possible_roots = FLATBUFFERS_MAX_ALIGNMENT / sizeof(uoffset_t) + 1; - possible_roots; possible_roots--) { - start -= sizeof(uoffset_t); - if (ReadScalar(start) + start == - reinterpret_cast(root)) - return start; - } - // We didn't find the root, either the "root" passed isn't really a root, - // or the buffer is corrupt. - // Assert, because calling this function with bad data may cause reads - // outside of buffer boundaries. - FLATBUFFERS_ASSERT(false); - return nullptr; -} - -/// @brief This return the prefixed size of a FlatBuffer. -inline uoffset_t GetPrefixedSize(const uint8_t *buf) { - return ReadScalar(buf); -} - -// Base class for native objects (FlatBuffer data de-serialized into native -// C++ data structures). -// Contains no functionality, purely documentative. -struct NativeTable {}; - -/// @brief Function types to be used with resolving hashes into objects and -/// back again. The resolver gets a pointer to a field inside an object API -/// object that is of the type specified in the schema using the attribute -/// `cpp_type` (it is thus important whatever you write to this address -/// matches that type). The value of this field is initially null, so you -/// may choose to implement a delayed binding lookup using this function -/// if you wish. The resolver does the opposite lookup, for when the object -/// is being serialized again. -typedef uint64_t hash_value_t; -// clang-format off -#ifdef FLATBUFFERS_CPP98_STL - typedef void (*resolver_function_t)(void **pointer_adr, hash_value_t hash); - typedef hash_value_t (*rehasher_function_t)(void *pointer); -#else - typedef std::function - resolver_function_t; - typedef std::function rehasher_function_t; -#endif -// clang-format on - -// Helper function to test if a field is present, using any of the field -// enums in the generated code. -// `table` must be a generated table type. Since this is a template parameter, -// this is not typechecked to be a subclass of Table, so beware! -// Note: this function will return false for fields equal to the default -// value, since they're not stored in the buffer (unless force_defaults was -// used). -template -bool IsFieldPresent(const T *table, typename T::FlatBuffersVTableOffset field) { - // Cast, since Table is a private baseclass of any table types. - return reinterpret_cast(table)->CheckField( - static_cast(field)); -} - -// Utility function for reverse lookups on the EnumNames*() functions -// (in the generated C++ code) -// names must be NULL terminated. -inline int LookupEnum(const char **names, const char *name) { - for (const char **p = names; *p; p++) - if (!strcmp(*p, name)) return static_cast(p - names); - return -1; -} - -// These macros allow us to layout a struct with a guarantee that they'll end -// up looking the same on different compilers and platforms. -// It does this by disallowing the compiler to do any padding, and then -// does padding itself by inserting extra padding fields that make every -// element aligned to its own size. -// Additionally, it manually sets the alignment of the struct as a whole, -// which is typically its largest element, or a custom size set in the schema -// by the force_align attribute. -// These are used in the generated code only. - -// clang-format off -#if defined(_MSC_VER) - #define FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(alignment) \ - __pragma(pack(1)) \ - struct __declspec(align(alignment)) - #define FLATBUFFERS_STRUCT_END(name, size) \ - __pragma(pack()) \ - static_assert(sizeof(name) == size, "compiler breaks packing rules") -#elif defined(__GNUC__) || defined(__clang__) || defined(__ICCARM__) - #define FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(alignment) \ - _Pragma("pack(1)") \ - struct __attribute__((aligned(alignment))) - #define FLATBUFFERS_STRUCT_END(name, size) \ - _Pragma("pack()") \ - static_assert(sizeof(name) == size, "compiler breaks packing rules") -#else - #error Unknown compiler, please define structure alignment macros -#endif -// clang-format on - -// Minimal reflection via code generation. -// Besides full-fat reflection (see reflection.h) and parsing/printing by -// loading schemas (see idl.h), we can also have code generation for mimimal -// reflection data which allows pretty-printing and other uses without needing -// a schema or a parser. -// Generate code with --reflect-types (types only) or --reflect-names (names -// also) to enable. -// See minireflect.h for utilities using this functionality. - -// These types are organized slightly differently as the ones in idl.h. -enum SequenceType { ST_TABLE, ST_STRUCT, ST_UNION, ST_ENUM }; - -// Scalars have the same order as in idl.h -// clang-format off -#define FLATBUFFERS_GEN_ELEMENTARY_TYPES(ET) \ - ET(ET_UTYPE) \ - ET(ET_BOOL) \ - ET(ET_CHAR) \ - ET(ET_UCHAR) \ - ET(ET_SHORT) \ - ET(ET_USHORT) \ - ET(ET_INT) \ - ET(ET_UINT) \ - ET(ET_LONG) \ - ET(ET_ULONG) \ - ET(ET_FLOAT) \ - ET(ET_DOUBLE) \ - ET(ET_STRING) \ - ET(ET_SEQUENCE) // See SequenceType. - -enum ElementaryType { - #define FLATBUFFERS_ET(E) E, - FLATBUFFERS_GEN_ELEMENTARY_TYPES(FLATBUFFERS_ET) - #undef FLATBUFFERS_ET -}; - -inline const char * const *ElementaryTypeNames() { - static const char * const names[] = { - #define FLATBUFFERS_ET(E) #E, - FLATBUFFERS_GEN_ELEMENTARY_TYPES(FLATBUFFERS_ET) - #undef FLATBUFFERS_ET - }; - return names; -} -// clang-format on - -// Basic type info cost just 16bits per field! -struct TypeCode { - uint16_t base_type : 4; // ElementaryType - uint16_t is_vector : 1; - int16_t sequence_ref : 11; // Index into type_refs below, or -1 for none. -}; - -static_assert(sizeof(TypeCode) == 2, "TypeCode"); - -struct TypeTable; - -// Signature of the static method present in each type. -typedef const TypeTable *(*TypeFunction)(); - -struct TypeTable { - SequenceType st; - size_t num_elems; // of type_codes, values, names (but not type_refs). - const TypeCode *type_codes; // num_elems count - const TypeFunction *type_refs; // less than num_elems entries (see TypeCode). - const int64_t *values; // Only set for non-consecutive enum/union or structs. - const char *const *names; // Only set if compiled with --reflect-names. -}; - -// String which identifies the current version of FlatBuffers. -// flatbuffer_version_string is used by Google developers to identify which -// applications uploaded to Google Play are using this library. This allows -// the development team at Google to determine the popularity of the library. -// How it works: Applications that are uploaded to the Google Play Store are -// scanned for this version string. We track which applications are using it -// to measure popularity. You are free to remove it (of course) but we would -// appreciate if you left it in. - -// Weak linkage is culled by VS & doesn't work on cygwin. -// clang-format off -#if !defined(_WIN32) && !defined(__CYGWIN__) - -extern volatile __attribute__((weak)) const char *flatbuffer_version_string; -volatile __attribute__((weak)) const char *flatbuffer_version_string = - "FlatBuffers " - FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MAJOR) "." - FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MINOR) "." - FLATBUFFERS_STRING(FLATBUFFERS_VERSION_REVISION); - -#endif // !defined(_WIN32) && !defined(__CYGWIN__) - -#define FLATBUFFERS_DEFINE_BITMASK_OPERATORS(E, T)\ - inline E operator | (E lhs, E rhs){\ - return E(T(lhs) | T(rhs));\ - }\ - inline E operator & (E lhs, E rhs){\ - return E(T(lhs) & T(rhs));\ - }\ - inline E operator ^ (E lhs, E rhs){\ - return E(T(lhs) ^ T(rhs));\ - }\ - inline E operator ~ (E lhs){\ - return E(~T(lhs));\ - }\ - inline E operator |= (E &lhs, E rhs){\ - lhs = lhs | rhs;\ - return lhs;\ - }\ - inline E operator &= (E &lhs, E rhs){\ - lhs = lhs & rhs;\ - return lhs;\ - }\ - inline E operator ^= (E &lhs, E rhs){\ - lhs = lhs ^ rhs;\ - return lhs;\ - }\ - inline bool operator !(E rhs) \ - {\ - return !bool(T(rhs)); \ - } -/// @endcond -} // namespace flatbuffers - -// clang-format on - -#endif // FLATBUFFERS_H_ diff --git a/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/stl_emulation.h b/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/stl_emulation.h deleted file mode 100644 index 8bae61bf..00000000 --- a/code/lib/tfmicro/third_party/flatbuffers/include/flatbuffers/stl_emulation.h +++ /dev/null @@ -1,307 +0,0 @@ -/* - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLATBUFFERS_STL_EMULATION_H_ -#define FLATBUFFERS_STL_EMULATION_H_ - -// clang-format off - -#include -#include -#include -#include -#include - -#if defined(_STLPORT_VERSION) && !defined(FLATBUFFERS_CPP98_STL) - #define FLATBUFFERS_CPP98_STL -#endif // defined(_STLPORT_VERSION) && !defined(FLATBUFFERS_CPP98_STL) - -#if defined(FLATBUFFERS_CPP98_STL) - #include -#endif // defined(FLATBUFFERS_CPP98_STL) - -// Check if we can use template aliases -// Not possible if Microsoft Compiler before 2012 -// Possible is the language feature __cpp_alias_templates is defined well -// Or possible if the C++ std is C+11 or newer -#if (defined(_MSC_VER) && _MSC_VER > 1700 /* MSVC2012 */) \ - || (defined(__cpp_alias_templates) && __cpp_alias_templates >= 200704) \ - || (defined(__cplusplus) && __cplusplus >= 201103L) - #define FLATBUFFERS_TEMPLATES_ALIASES -#endif - -// This header provides backwards compatibility for C++98 STLs like stlport. -namespace flatbuffers { - -// Retrieve ::back() from a string in a way that is compatible with pre C++11 -// STLs (e.g stlport). -inline char& string_back(std::string &value) { - return value[value.length() - 1]; -} - -inline char string_back(const std::string &value) { - return value[value.length() - 1]; -} - -// Helper method that retrieves ::data() from a vector in a way that is -// compatible with pre C++11 STLs (e.g stlport). -template inline T *vector_data(std::vector &vector) { - // In some debug environments, operator[] does bounds checking, so &vector[0] - // can't be used. - return vector.empty() ? nullptr : &vector[0]; -} - -template inline const T *vector_data( - const std::vector &vector) { - return vector.empty() ? nullptr : &vector[0]; -} - -template -inline void vector_emplace_back(std::vector *vector, V &&data) { - #if defined(FLATBUFFERS_CPP98_STL) - vector->push_back(data); - #else - vector->emplace_back(std::forward(data)); - #endif // defined(FLATBUFFERS_CPP98_STL) -} - -#ifndef FLATBUFFERS_CPP98_STL - #if defined(FLATBUFFERS_TEMPLATES_ALIASES) - template - using numeric_limits = std::numeric_limits; - #else - template class numeric_limits : - public std::numeric_limits {}; - #endif // defined(FLATBUFFERS_TEMPLATES_ALIASES) -#else - template class numeric_limits : - public std::numeric_limits { - public: - // Android NDK fix. - static T lowest() { - return std::numeric_limits::min(); - } - }; - - template <> class numeric_limits : - public std::numeric_limits { - public: - static float lowest() { return -FLT_MAX; } - }; - - template <> class numeric_limits : - public std::numeric_limits { - public: - static double lowest() { return -DBL_MAX; } - }; - - template <> class numeric_limits { - public: - static unsigned long long min() { return 0ULL; } - static unsigned long long max() { return ~0ULL; } - static unsigned long long lowest() { - return numeric_limits::min(); - } - }; - - template <> class numeric_limits { - public: - static long long min() { - return static_cast(1ULL << ((sizeof(long long) << 3) - 1)); - } - static long long max() { - return static_cast( - (1ULL << ((sizeof(long long) << 3) - 1)) - 1); - } - static long long lowest() { - return numeric_limits::min(); - } - }; -#endif // FLATBUFFERS_CPP98_STL - -#if defined(FLATBUFFERS_TEMPLATES_ALIASES) - #ifndef FLATBUFFERS_CPP98_STL - template using is_scalar = std::is_scalar; - template using is_same = std::is_same; - template using is_floating_point = std::is_floating_point; - template using is_unsigned = std::is_unsigned; - template using is_enum = std::is_enum; - template using make_unsigned = std::make_unsigned; - template - using conditional = std::conditional; - template - using integral_constant = std::integral_constant; - #else - // Map C++ TR1 templates defined by stlport. - template using is_scalar = std::tr1::is_scalar; - template using is_same = std::tr1::is_same; - template using is_floating_point = - std::tr1::is_floating_point; - template using is_unsigned = std::tr1::is_unsigned; - template using is_enum = std::tr1::is_enum; - // Android NDK doesn't have std::make_unsigned or std::tr1::make_unsigned. - template struct make_unsigned { - static_assert(is_unsigned::value, "Specialization not implemented!"); - using type = T; - }; - template<> struct make_unsigned { using type = unsigned char; }; - template<> struct make_unsigned { using type = unsigned short; }; - template<> struct make_unsigned { using type = unsigned int; }; - template<> struct make_unsigned { using type = unsigned long; }; - template<> - struct make_unsigned { using type = unsigned long long; }; - template - using conditional = std::tr1::conditional; - template - using integral_constant = std::tr1::integral_constant; - #endif // !FLATBUFFERS_CPP98_STL -#else - // MSVC 2010 doesn't support C++11 aliases. - template struct is_scalar : public std::is_scalar {}; - template struct is_same : public std::is_same {}; - template struct is_floating_point : - public std::is_floating_point {}; - template struct is_unsigned : public std::is_unsigned {}; - template struct is_enum : public std::is_enum {}; - template struct make_unsigned : public std::make_unsigned {}; - template - struct conditional : public std::conditional {}; - template - struct integral_constant : public std::integral_constant {}; -#endif // defined(FLATBUFFERS_TEMPLATES_ALIASES) - -#ifndef FLATBUFFERS_CPP98_STL - #if defined(FLATBUFFERS_TEMPLATES_ALIASES) - template using unique_ptr = std::unique_ptr; - #else - // MSVC 2010 doesn't support C++11 aliases. - // We're manually "aliasing" the class here as we want to bring unique_ptr - // into the flatbuffers namespace. We have unique_ptr in the flatbuffers - // namespace we have a completely independent implemenation (see below) - // for C++98 STL implementations. - template class unique_ptr : public std::unique_ptr { - public: - unique_ptr() {} - explicit unique_ptr(T* p) : std::unique_ptr(p) {} - unique_ptr(std::unique_ptr&& u) { *this = std::move(u); } - unique_ptr(unique_ptr&& u) { *this = std::move(u); } - unique_ptr& operator=(std::unique_ptr&& u) { - std::unique_ptr::reset(u.release()); - return *this; - } - unique_ptr& operator=(unique_ptr&& u) { - std::unique_ptr::reset(u.release()); - return *this; - } - unique_ptr& operator=(T* p) { - return std::unique_ptr::operator=(p); - } - }; - #endif // defined(FLATBUFFERS_TEMPLATES_ALIASES) -#else - // Very limited implementation of unique_ptr. - // This is provided simply to allow the C++ code generated from the default - // settings to function in C++98 environments with no modifications. - template class unique_ptr { - public: - typedef T element_type; - - unique_ptr() : ptr_(nullptr) {} - explicit unique_ptr(T* p) : ptr_(p) {} - unique_ptr(unique_ptr&& u) : ptr_(nullptr) { reset(u.release()); } - unique_ptr(const unique_ptr& u) : ptr_(nullptr) { - reset(const_cast(&u)->release()); - } - ~unique_ptr() { reset(); } - - unique_ptr& operator=(const unique_ptr& u) { - reset(const_cast(&u)->release()); - return *this; - } - - unique_ptr& operator=(unique_ptr&& u) { - reset(u.release()); - return *this; - } - - unique_ptr& operator=(T* p) { - reset(p); - return *this; - } - - const T& operator*() const { return *ptr_; } - T* operator->() const { return ptr_; } - T* get() const noexcept { return ptr_; } - explicit operator bool() const { return ptr_ != nullptr; } - - // modifiers - T* release() { - T* value = ptr_; - ptr_ = nullptr; - return value; - } - - void reset(T* p = nullptr) { - T* value = ptr_; - ptr_ = p; - if (value) delete value; - } - - void swap(unique_ptr& u) { - T* temp_ptr = ptr_; - ptr_ = u.ptr_; - u.ptr_ = temp_ptr; - } - - private: - T* ptr_; - }; - - template bool operator==(const unique_ptr& x, - const unique_ptr& y) { - return x.get() == y.get(); - } - - template bool operator==(const unique_ptr& x, - const D* y) { - return static_cast(x.get()) == y; - } - - template bool operator==(const unique_ptr& x, intptr_t y) { - return reinterpret_cast(x.get()) == y; - } - - template bool operator!=(const unique_ptr& x, decltype(nullptr)) { - return !!x; - } - - template bool operator!=(decltype(nullptr), const unique_ptr& x) { - return !!x; - } - - template bool operator==(const unique_ptr& x, decltype(nullptr)) { - return !x; - } - - template bool operator==(decltype(nullptr), const unique_ptr& x) { - return !x; - } - -#endif // !FLATBUFFERS_CPP98_STL - -} // namespace flatbuffers - -#endif // FLATBUFFERS_STL_EMULATION_H_ diff --git a/code/lib/tfmicro/third_party/gemmlowp/fixedpoint/fixedpoint.h b/code/lib/tfmicro/third_party/gemmlowp/fixedpoint/fixedpoint.h deleted file mode 100644 index 51b5aff4..00000000 --- a/code/lib/tfmicro/third_party/gemmlowp/fixedpoint/fixedpoint.h +++ /dev/null @@ -1,900 +0,0 @@ -// Copyright 2015 The Gemmlowp Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// fixedpoint.h: fixed-point arithmetic, with basic operations and -// a few math functions such as tanh. - -#ifndef GEMMLOWP_INTERNAL_FIXEDPOINT_H_ -#define GEMMLOWP_INTERNAL_FIXEDPOINT_H_ - -#include -#include -#include -#include -#include - -#include "../internal/detect_platform.h" - -namespace gemmlowp { - -// Part 1: Low-level integer-arithmetic primitives. -// The implementations here are generic implementations valid for -// scalar types (e.g. std::int32_t). Architecture-specific SIMD types -// (e.g. NEON int32x4_t) may be supported by providing -// specializations for them in separate files. -// -// The purpose of these primitives is two-fold: -// - They will be used to implement higher-level fixed-point -// abstractions, namely the FixedPoint class and its arithmetic -// operators. -// - They will be directly used to implement some more involved -// fixed-point computations, e.g. the fixed-point implementation -// of math functions such as tanh. - -// Some compile-time traits around raw types to handle SIMD aspects: -// number of lanes, underlying scalar type. -template -struct FixedPointRawTypeTraits {}; - -template <> -struct FixedPointRawTypeTraits { - typedef std::int32_t ScalarRawType; - static constexpr int kLanes = 1; -}; - -template <> -struct FixedPointRawTypeTraits { - typedef std::int16_t ScalarRawType; - static constexpr int kLanes = 1; -}; - -// Returns a SIMD value duplicating a scalar value across all lanes. -template -tRawType Dup(typename FixedPointRawTypeTraits::ScalarRawType x) { - return x; -} - -// Plain bit-wise AND -template -tIntegerType BitAnd(tIntegerType a, tIntegerType b) { - return a & b; -} - -// Plain bit-wise OR -template -tIntegerType BitOr(tIntegerType a, tIntegerType b) { - return a | b; -} - -// Plain bit-wise XOR -template -tIntegerType BitXor(tIntegerType a, tIntegerType b) { - return a ^ b; -} - -// Plain bit-wise NOT -template -tIntegerType BitNot(tIntegerType a) { - return ~a; -} - -// Integer addition. Not saturating. Overflow is undefined behavior. -template -tIntegerType Add(tIntegerType a, tIntegerType b) { - return a + b; -} - -// Integer subtraction. Not saturating. Overflow is undefined behavior. -template -tIntegerType Mul(tIntegerType a, tIntegerType b) { - return a * b; -} - -template -tIntegerType Sub(tIntegerType a, tIntegerType b) { - return a - b; -} - -// Integer unary negative. Not saturating. Overflow is undefined behavior. -template -tIntegerType Neg(tIntegerType a) { - return -a; -} - -// Integer arithmetic left-shift, equivalent to multiplying with a power of two. -// Negative values are OK. In case of overflow, no Undefined -// Behavior, but the results are implementation-defined (in practice, -// they currently are saturated, but we make no commitment to that). The idea -// is that the caller will want to implement the overflowing cases with -// saturation with compare-and-mask, so we don't care about the results -// in the overflow case, we just want to avoid undefined behavior. -// -// tIntegerType may be int32 or any narrower signed type. -template -tIntegerType ShiftLeft(tIntegerType a, int offset) { - const std::int64_t wide_a = static_cast(a); - const std::int64_t wide_shifted = wide_a * (1 << offset); - const auto min = std::numeric_limits::min(); - const auto max = std::numeric_limits::max(); - return wide_shifted < min - ? min - : wide_shifted > max ? max - : static_cast(wide_shifted); -} - -// Integer arithmetic right-shift. Not rounding. -// Relying on implementation-defined, but in-practice-consistent, -// C++ compiler behavior. -template -tIntegerType ShiftRight(tIntegerType a, int offset) { - return a >> offset; -} - -// Each bit of the result is set to the corresponding bit of either then_val or -// else_val depending on whether the corresponding bit of if_mask is set. -// Equivalent to the VBSL instruction in ARM NEON. -template -tIntegerType SelectUsingMask(tIntegerType if_mask, tIntegerType then_val, - tIntegerType else_val) { - return BitXor(BitAnd(if_mask, then_val), BitAnd(BitNot(if_mask), else_val)); -} - -// For each input scalar, the corresponding bits of the result are set if the -// input scalar is non-zero. -template -tIntegerType MaskIfNonZero(tIntegerType a) { - static constexpr tIntegerType zero = 0; - return a ? BitNot(zero) : zero; -} - -// For each input scalar, the corresponding bits of the result are set if the -// input scalar is zero. -template -tIntegerType MaskIfZero(tIntegerType a) { - return MaskIfNonZero(!a); -} - -// For each pair of input scalars, the corresponding bits of the result are -// set if the input scalars are equal. -template -tIntegerType MaskIfEqual(tIntegerType a, tIntegerType b) { - return MaskIfNonZero(a == b); -} - -// For each pair of input scalars, the corresponding bits of the result are -// set if the input scalars are not equal. -template -tIntegerType MaskIfNotEqual(tIntegerType a, tIntegerType b) { - return MaskIfNonZero(a != b); -} - -// For each pair of input scalars, the corresponding bits of the result are -// set if the input scalars a, b satisfy a > b. -template -tIntegerType MaskIfGreaterThan(tIntegerType a, tIntegerType b) { - return MaskIfNonZero(a > b); -} - -// For each pair of input scalars, the corresponding bits of the result are -// set if the input scalars a, b satisfy a >= b. -template -tIntegerType MaskIfGreaterThanOrEqual(tIntegerType a, tIntegerType b) { - return MaskIfNonZero(a >= b); -} - -// For each pair of input scalars, the corresponding bits of the result are -// set if the input scalars a, b satisfy a < b. -template -tIntegerType MaskIfLessThan(tIntegerType a, tIntegerType b) { - return MaskIfNonZero(a < b); -} - -// For each pair of input scalars, the corresponding bits of the result are -// set if the input scalars a, b satisfy a <= b. -template -tIntegerType MaskIfLessThanOrEqual(tIntegerType a, tIntegerType b) { - return MaskIfNonZero(a <= b); -} - -// Returns true if all of the input scalars are nonzero. -// This function may currently assume that each of the input scalars has either -// all or none of its bits set. Otherwise, its behavior is currently undefined. -template -bool All(tIntegerType a) { - return a; -} - -// Returns true if any of the input scalars are nonzero. -// This function may currently assume that each of the input scalars has either -// all or none of its bits set. Otherwise, its behavior is currently undefined. -template -bool Any(tIntegerType a) { - return a; -} - -// Returns (a+b)/2, rounded to the nearest integer. -// Equivalent to VRHADD in the ARM NEON instruction set. -template -IntegerType RoundingHalfSum(IntegerType a, IntegerType b) { - static_assert(std::is_same::value, "unimplemented"); - (void)b; - return a; -} - -template <> -inline std::int32_t RoundingHalfSum(std::int32_t a, std::int32_t b) { - std::int64_t a64 = a; - std::int64_t b64 = b; - std::int64_t sum = a64 + b64; - std::int64_t sign = sum >= 0 ? 1 : -1; - return static_cast((sum + sign) / 2); -} - -template <> -inline std::int16_t RoundingHalfSum(std::int16_t a, std::int16_t b) { - std::int32_t a32 = a; - std::int32_t b32 = b; - std::int32_t sum = a32 + b32; - std::int32_t sign = sum >= 0 ? 1 : -1; - return static_cast((sum + sign) / 2); -} - -template -IntegerType SaturatingAdd(IntegerType a, IntegerType b) { - static_assert(std::is_same::value, "unimplemented"); - (void)b; - return a; -} - -// So far this is only needed for int16. -template <> -inline std::int16_t SaturatingAdd(std::int16_t a, std::int16_t b) { - std::int32_t a32 = a; - std::int32_t b32 = b; - std::int32_t sum = a32 + b32; - return static_cast( - std::min(static_cast(32767), - std::max(static_cast(-32768), sum))); -} - -// Returns a+b, saturating if the integers are 16bit or narrower, -// otherwise just a plain addition. -template -struct AddSaturatingIf16BitImpl { - static IntegerType Run(IntegerType a, IntegerType b) { return Add(a, b); } -}; -template -struct AddSaturatingIf16BitImpl { - static IntegerType Run(IntegerType a, IntegerType b) { - return SaturatingAdd(a, b); - } -}; -template -IntegerType AddSaturatingIf16Bit(IntegerType a, IntegerType b) { - using ScalarType = - typename FixedPointRawTypeTraits::ScalarRawType; - return AddSaturatingIf16BitImpl::Run(a, - b); -} - -// Returns the integer that represents the product of two fixed-point -// numbers, interpreting all integers as fixed-point values in the -// interval [-1, 1), rounding to the nearest value, and saturating -// -1 * -1 to the maximum value (since 1 is not in the half-open -// interval [-1, 1)). -// -// [The explanation below specializes to std::int32_t for example purpose.] -// -// The mapping between IntegerType and the interval [-1, 1) is unique and -// implied by IntegerType, which is assumed to be signed. For example, -// for IntegerType==std::int32_t, the mapping is -// real_value = integer_value / 2^31. -// So in this case, and leaving aside rounding and saturating, this -// function computes ((a / 2^31) * (b / 2^31)) * 2^31, which simplifies to -// (a * b) / 2^31. -// -// The 'doubling' part in the name of this function comes from the fact that -// this operation is very close to a "multiply-high" operation, keeping only -// the top half bits, except that that would be effectively computing -// (a * b) / 2^32, -// so here we are computing 2x that, since -// 1/2^31 = 2 * 1/2^32. -// The idea is to use all of the available 32 bits in the destination int32 -// value. -// -// [End of the explanation specializing to int32.] -// -// This is equivalent to the VQRDMULH instruction in ARM NEON. -template -IntegerType SaturatingRoundingDoublingHighMul(IntegerType a, IntegerType b) { - static_assert(std::is_same::value, "unimplemented"); - (void)b; - return a; -} - -// This function implements the same computation as the ARMv7 NEON VQRDMULH -// instruction. -template <> -inline std::int32_t SaturatingRoundingDoublingHighMul(std::int32_t a, - std::int32_t b) { - bool overflow = a == b && a == std::numeric_limits::min(); - std::int64_t a_64(a); - std::int64_t b_64(b); - std::int64_t ab_64 = a_64 * b_64; - std::int32_t nudge = ab_64 >= 0 ? (1 << 30) : (1 - (1 << 30)); - std::int32_t ab_x2_high32 = - static_cast((ab_64 + nudge) / (1ll << 31)); - return overflow ? std::numeric_limits::max() : ab_x2_high32; -} - -template <> -inline std::int16_t SaturatingRoundingDoublingHighMul(std::int16_t a, - std::int16_t b) { - bool overflow = a == b && a == std::numeric_limits::min(); - std::int32_t a_32(a); - std::int32_t b_32(b); - std::int32_t ab_32 = a_32 * b_32; - std::int16_t nudge = ab_32 >= 0 ? (1 << 14) : (1 - (1 << 14)); - std::int16_t ab_x2_high16 = - static_cast((ab_32 + nudge) / (1 << 15)); - return overflow ? std::numeric_limits::max() : ab_x2_high16; -} - -// Correctly-rounded-to-nearest division by a power-of-two. -// Also known as a rounding arithmetic right shift. -template -inline IntegerType RoundingDivideByPOT(IntegerType x, int exponent) { - assert(exponent >= 0); - assert(exponent <= 31); - const IntegerType mask = Dup((1ll << exponent) - 1); - const IntegerType zero = Dup(0); - const IntegerType one = Dup(1); - const IntegerType remainder = BitAnd(x, mask); - const IntegerType threshold = - Add(ShiftRight(mask, 1), BitAnd(MaskIfLessThan(x, zero), one)); - return Add(ShiftRight(x, exponent), - BitAnd(MaskIfGreaterThan(remainder, threshold), one)); -} - -// Returns the product of a run-time integer value by a compile-time power -// of two, with either a positive exponent (equivalent to an arithmetic -// left shift, saturating) or a negative exponent (equivalent to an arithmetic -// right shift, rounding to nearest). -template 0 ? 1 : Exponent < 0 ? -1 : 0)> -struct ImplSaturatingRoundingMultiplyByPOT {}; - -template -struct ImplSaturatingRoundingMultiplyByPOT { - static IntegerType eval(IntegerType x) { return x; } -}; - -template -struct ImplSaturatingRoundingMultiplyByPOT { - static IntegerType eval(IntegerType x) { - using ScalarIntegerType = - typename FixedPointRawTypeTraits::ScalarRawType; - const IntegerType min = - Dup(std::numeric_limits::min()); - const IntegerType max = - Dup(std::numeric_limits::max()); - const int ScalarIntegerTypeBits = 8 * sizeof(ScalarIntegerType); - - const std::int32_t threshold = - ((1 << (ScalarIntegerTypeBits - 1 - Exponent)) - 1); - const IntegerType positive_mask = - MaskIfGreaterThan(x, Dup(threshold)); - const IntegerType negative_mask = - MaskIfLessThan(x, Dup(-threshold)); - - IntegerType result = ShiftLeft(x, Exponent); - result = SelectUsingMask(positive_mask, max, result); - result = SelectUsingMask(negative_mask, min, result); - return result; - } -}; - -template -struct ImplSaturatingRoundingMultiplyByPOT { - static IntegerType eval(IntegerType x) { - return RoundingDivideByPOT(x, -Exponent); - } -}; - -template -IntegerType SaturatingRoundingMultiplyByPOT(IntegerType x) { - return ImplSaturatingRoundingMultiplyByPOT::eval(x); -} - -// Part 2: the FixedPoint class. - -// A FixedPoint object represents a fixed-point value stored in the underlying -// integer type tRawType, if tRawType is a plain scalar integer type. -// Alternatively, tRawType may be a SIMD type (e.g. NEON int32x4_t) in which -// case a FixedPoint object represents a corresponding SIMD vector of fixed -// point values. -// -// tIntegerBits describes the range of the fixed-point format: if -// tIntegerBits == m then the range of representable values is the half-open -// interval [-2^m; 2^m) where the open boundary on the right side means that -// 2^m is not representable (how close the maximum representable value is to -// it, depends on bit-depth of tRawType). -// -// In "Q format notation", -// https://en.wikipedia.org/wiki/Q_(number_format) -// we are describing the format -// Qm.n -// where -// m = tIntegerBits -// and -// n = NumberOfBits(tRawType) - (m + 1) -// Note that the (m + 1) in the above line is because we adopt the convention -// that we count the integer bits exclusively of the sign bit; so (m + 1) is -// the total number of integer bits inclusive of the sign bit. -// -// Accordingly, the number of integral representable values in our range -// [-2^m ; 2^m) -// is equal to 2^(m+1). -template -class FixedPoint { - public: - typedef tRawType RawType; - - typedef FixedPointRawTypeTraits RawTypeTraits; - typedef typename RawTypeTraits::ScalarRawType ScalarRawType; - - static constexpr int kTotalBits = 8 * sizeof(ScalarRawType); - static constexpr int kIntegerBits = tIntegerBits; - static constexpr int kFractionalBits = kTotalBits - 1 - kIntegerBits; - static_assert(kIntegerBits >= 0 && kIntegerBits < kTotalBits, - "bad IntegerBits"); - - typedef FixedPoint ScalarFixedPointType; - - static const ScalarRawType ScalarRawMin() { - return std::numeric_limits::min(); - } - - static const ScalarRawType ScalarRawMax() { - return std::numeric_limits::max(); - } - - static const ScalarRawType RawMin() { - return VectorFromScalar(ScalarRawMin()); - } - - static const ScalarRawType RawMax() { - return VectorFromScalar(ScalarRawMax()); - } - - static FixedPoint FromRaw(RawType x) { - FixedPoint retval; - retval.raw() = x; - return retval; - } - - static FixedPoint FromScalarRaw(ScalarRawType x) { - FixedPoint retval; - retval.raw() = Dup(x); - return retval; - } - - static FixedPoint FromScalarFixedPoint(ScalarFixedPointType x) { - return FromScalarRaw(x.raw()); - } - - template - static FixedPoint ConstantPOT() { - static constexpr int kOffset = kFractionalBits + Exponent; - static_assert( - kOffset < 31, - "Constant not exactly representable in this fixed-point format"); - return FromScalarRaw(ScalarRawType(1) << kOffset); - } - - static FixedPoint Zero() { return FromScalarRaw(0); } - - static FixedPoint One() { - return FromScalarRaw( - kIntegerBits == 0 - ? ScalarRawMax() - : (ScalarRawType(1) << (kIntegerBits == 0 ? 0 : kFractionalBits))); - } - - static FixedPoint FromDouble(double x) { - const double min_bound = static_cast(ScalarRawMin()); - const double max_bound = static_cast(ScalarRawMax()); - return FromScalarRaw(static_cast(std::min( - std::max(round(x * static_cast(1ll << kFractionalBits)), - min_bound), - max_bound))); - } - - RawType raw() const { return i_; } - RawType& raw() { return i_; } - - private: - RawType i_; -}; - -// Part 3: implementation of arithmetic operators for the -// FixedPoint class, and a few related functions. - -// A FixedPoint multiplication is just a -// SaturatingRoundingDoublingHighMul operation on the underlying -// raw integer values. The IntegerBits simply add up, as is obvious -// from the fact that the range is [-2^IntegerBits, 2^IntegerBits). -template -FixedPoint operator*( - FixedPoint a, - FixedPoint b) { - FixedPoint c; - c.raw() = SaturatingRoundingDoublingHighMul(a.raw(), b.raw()); - return c; -} - -// Tweaking IntegerBits gives exact multiplication by a power of two. -template -FixedPoint ExactMulByPot( - FixedPoint a) { - FixedPoint c; - c.raw() = a.raw(); - return c; -} - -// If we want to leave IntegerBits fixed, then multiplication -// by a power of two has to be saturating/rounding, not exact anymore. -template -FixedPoint SaturatingRoundingMultiplyByPOT( - FixedPoint a) { - return FixedPoint::FromRaw( - SaturatingRoundingMultiplyByPOT(a.raw())); -} - -// Generic arithmetic operators. - -#define MAKE_FIXEDPOINT_UNARY_FUNC(FuncName, ImplFuncName) \ - template \ - FixedPoint FuncName( \ - FixedPoint a) { \ - return FixedPoint::FromRaw(ImplFuncName(a.raw())); \ - } - -#define MAKE_FIXEDPOINT_BINARY_FUNC(FuncName, ImplFuncName) \ - template \ - FixedPoint FuncName( \ - FixedPoint a, \ - FixedPoint b) { \ - return FixedPoint::FromRaw( \ - ImplFuncName(a.raw(), b.raw())); \ - } - -MAKE_FIXEDPOINT_UNARY_FUNC(operator-, Neg) -MAKE_FIXEDPOINT_UNARY_FUNC(operator~, BitNot) -MAKE_FIXEDPOINT_BINARY_FUNC(operator+, Add) -MAKE_FIXEDPOINT_BINARY_FUNC(operator-, Sub) -MAKE_FIXEDPOINT_BINARY_FUNC(operator&, BitAnd) -MAKE_FIXEDPOINT_BINARY_FUNC(operator^, BitXor) -MAKE_FIXEDPOINT_BINARY_FUNC(operator|, BitOr) -MAKE_FIXEDPOINT_BINARY_FUNC(RoundingHalfSum, RoundingHalfSum) - -#undef MAKE_FIXEDPOINT_UNARY_FUNC -#undef MAKE_FIXEDPOINT_BINARY_FUNC - -#define MAKE_FIXEDPOINT_UNARY_FUNC_RETURNING_RAW(FuncName) \ - template \ - tRawType FuncName(FixedPoint a) { \ - return FuncName(a.raw()); \ - } - -#define MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(FuncName) \ - template \ - tRawType FuncName(FixedPoint a, \ - FixedPoint b) { \ - return FuncName(a.raw(), b.raw()); \ - } - -MAKE_FIXEDPOINT_UNARY_FUNC_RETURNING_RAW(MaskIfZero) -MAKE_FIXEDPOINT_UNARY_FUNC_RETURNING_RAW(MaskIfNonZero) -MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfEqual) -MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfNotEqual) -MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfGreaterThan) -MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfGreaterThanOrEqual) -MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfLessThan) -MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW(MaskIfLessThanOrEqual) - -#undef MAKE_FIXEDPOINT_UNARY_FUNC_RETURNING_RAW -#undef MAKE_FIXEDPOINT_BINARY_FUNC_RETURNING_RAW - -template -FixedPoint SelectUsingMask( - tRawType if_mask, FixedPoint then_val, - FixedPoint else_val) { - return FixedPoint::FromRaw( - SelectUsingMask(if_mask, then_val.raw(), else_val.raw())); -} - -template -bool operator==(FixedPoint a, - FixedPoint b) { - return All(MaskIfEqual(a.raw(), b.raw())); -} - -template -bool operator!=(FixedPoint a, - FixedPoint b) { - return !(a == b); -} - -template -FixedPoint SaturatingAdd( - FixedPoint a, - FixedPoint b) { - return FixedPoint::FromRaw( - SaturatingAdd(a.raw(), b.raw())); -} - -template -FixedPoint AddSaturatingIf16Bit( - FixedPoint a, - FixedPoint b) { - return FixedPoint::FromRaw( - AddSaturatingIf16Bit(a.raw(), b.raw())); -} - -// Conversion to floating-point. -template -double ToDouble(FixedPoint x) { - static_assert(FixedPointRawTypeTraits::kLanes == 1, - "not applicable to SIMD types"); - typedef FixedPoint F; - return x.raw() / static_cast(1ll << F::kFractionalBits); -} - -// Rescale changes the number of IntegerBits and updates the underlying -// raw integer value accordingly. -template -FixedPoint Rescale( - FixedPoint x) { - static constexpr int kExponent = tIntegerBitsSrc - tIntegerBitsDst; - FixedPoint result; - result.raw() = SaturatingRoundingMultiplyByPOT(x.raw()); - return result; -} - -// CheckedFixedPointConstant allows to specify fixed-point constants -// initialized as real numbers, in a way that does not compile floating-point -// arithmetic in production code, yet still checks agreement with the -// floating-point expressions when asserts are enabled. -// -// The raw integer value provided is always a int32, encoding a 32-bit -// fixed-point value, regardless of the actual Scalar type. This allows -// writing generic code that applies just as well to the 32-bit and 16-bit -// cases. In the 16-bit case, the raw integer value is internally -// rounding-shifted by 16 bits to the right. -template -inline typename FixedPointType::ScalarRawType RescaleConstantInitializer( - std::int32_t int32_value) { - typedef typename FixedPointType::ScalarRawType ScalarRawType; - static constexpr int ScalarTypeBits = 8 * sizeof(ScalarRawType); - return static_cast( - RoundingDivideByPOT(int32_value, 32 - ScalarTypeBits)); -} -#ifdef GEMMLOWP_ENABLE_FIXEDPOINT_CONSTANTS_CHECKS -template -FixedPointType CheckedFixedPointConstant(std::int32_t raw_value, - double double_value) { - const FixedPointType result = FixedPointType::FromScalarRaw(raw_value); - assert(result == FixedPointType::FromDouble(double_value)); - return result; -} -#define GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(FixedPointType, \ - ScalarRawInt32Value, DoubleValue) \ - (gemmlowp::CheckedFixedPointConstant( \ - gemmlowp::RescaleConstantInitializer( \ - ScalarRawInt32Value), \ - DoubleValue)) - -#else -#define GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(FixedPointType, \ - ScalarRawInt32Value, DoubleValue) \ - (FixedPointType::FromScalarRaw( \ - gemmlowp::RescaleConstantInitializer( \ - ScalarRawInt32Value))) -#endif - -// Implementation of exponential function. - -// Returns exp(x) for x in [-1/4, 0). -template -FixedPoint exp_on_interval_between_negative_one_quarter_and_0_excl( - FixedPoint a) { - typedef FixedPoint F; - const F constant_term = - GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F, 1895147668, std::exp(-1.0 / 8.0)); - const F constant_1_over_3 = - GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F, 715827883, 1.0 / 3.0); - // We're evaluating a Taylor expansion around -1/8, so we do the change of - // variable: x = a + 1/8. - // In fixed-point with 0 integer bits, 1/8 is represented by 1 << 28. - F x = a + F::template ConstantPOT<-3>(); - F x2 = x * x; - F x3 = x2 * x; - F x4 = x2 * x2; - F x4_over_4 = SaturatingRoundingMultiplyByPOT<-2>(x4); - F x4_over_24_plus_x3_over_6_plus_x2_over_2 = - SaturatingRoundingMultiplyByPOT<-1>( - ((x4_over_4 + x3) * constant_1_over_3) + x2); - return AddSaturatingIf16Bit( - constant_term, - constant_term * (x + x4_over_24_plus_x3_over_6_plus_x2_over_2)); -} - -// Returns exp(x) for x < 0. -template -FixedPoint exp_on_negative_values( - FixedPoint a) { - typedef FixedPoint InputF; - typedef FixedPoint ResultF; - static constexpr int kFractionalBits = InputF::kFractionalBits; - static constexpr int kIntegerBits = InputF::kIntegerBits; - const InputF kOneQuarter = InputF::template ConstantPOT<-2>(); - InputF mask = kOneQuarter - InputF::FromScalarRaw(1); - InputF a_mod_quarter_minus_one_quarter = (a & mask) - kOneQuarter; - ResultF result = exp_on_interval_between_negative_one_quarter_and_0_excl( - Rescale<0>(a_mod_quarter_minus_one_quarter)); - tRawType remainder = (a_mod_quarter_minus_one_quarter - a).raw(); - -#define GEMMLOWP_EXP_BARREL_SHIFTER(Exponent, FixedPointMultiplier) \ - if (kIntegerBits > Exponent) { \ - const ResultF kMultiplier = GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT( \ - ResultF, FixedPointMultiplier, std::exp(-std::pow(2.0, Exponent))); \ - static constexpr int kShiftAmount = \ - kIntegerBits > Exponent ? kFractionalBits + Exponent : 0; \ - result = SelectUsingMask( \ - MaskIfNonZero(BitAnd(remainder, Dup(1 << kShiftAmount))), \ - result * kMultiplier, result); \ - } - - GEMMLOWP_EXP_BARREL_SHIFTER(-2, 1672461947); - GEMMLOWP_EXP_BARREL_SHIFTER(-1, 1302514674); - GEMMLOWP_EXP_BARREL_SHIFTER(+0, 790015084); - GEMMLOWP_EXP_BARREL_SHIFTER(+1, 290630308); - GEMMLOWP_EXP_BARREL_SHIFTER(+2, 39332535); - GEMMLOWP_EXP_BARREL_SHIFTER(+3, 720401); - GEMMLOWP_EXP_BARREL_SHIFTER(+4, 242); - -#undef GEMMLOWP_EXP_BARREL_SHIFTER - - static constexpr int clampB = kIntegerBits > 5 ? 36 - kIntegerBits : 0; - if (kIntegerBits > 5) { - const InputF clamp = - GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(InputF, -(1 << clampB), -32.0); - result = SelectUsingMask(MaskIfLessThan(a, clamp), ResultF::Zero(), result); - } - - result = SelectUsingMask(MaskIfZero(a), ResultF::One(), result); - return result; -} - -// Implementation of tanh: (1 - exp(-2x)) / (1 + exp(-2x)). - -// Returns (1 - x) / (1 + x) for x in (0, 1). -template -FixedPoint one_minus_x_over_one_plus_x_for_x_in_0_1( - FixedPoint a) { - typedef FixedPoint F0; - typedef FixedPoint F2; - F0 half_denominator = RoundingHalfSum(a, F0::One()); - // Newton-Raphson division - // https://en.wikipedia.org/wiki/Division_algorithm#Newton.E2.80.93Raphson_division - // Refer to that page for the logic behind the 48/17 and 32/17 constants. - const F2 constant_48_over_17 = - GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F2, 1515870810, 48.0 / 17.0); - const F2 constant_neg_32_over_17 = - GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F2, -1010580540, -32.0 / 17.0); - F2 x = constant_48_over_17 + half_denominator * constant_neg_32_over_17; - for (int i = 0; i < 3; i++) { - F2 half_denominator_times_x = half_denominator * x; - F2 one_minus_half_denominator_times_x = - F2::One() - half_denominator_times_x; - x = x + Rescale<2>(x * one_minus_half_denominator_times_x); - } - return Rescale<0>(x - F2::One()); -} - -// Returns -tanh(x) for x < 0. -template -FixedPoint neg_tanh_on_negative_values( - FixedPoint a) { - return one_minus_x_over_one_plus_x_for_x_in_0_1( - exp_on_negative_values(ExactMulByPot<1>(a))); -} - -// Returns tanh(x) for any x. -template -FixedPoint tanh(FixedPoint a) { - typedef FixedPoint InputF; - typedef FixedPoint ResultF; - tRawType mask_if_negative = MaskIfLessThan(a, InputF::Zero()); - tRawType mask_if_zero = MaskIfZero(a); - InputF n = SelectUsingMask(mask_if_negative, a, -a); - ResultF t = neg_tanh_on_negative_values(n); - return SelectUsingMask(mask_if_zero, ResultF::Zero(), - SelectUsingMask(mask_if_negative, -t, t)); -} - -// Implementation of logistic function. - -// Returns 1 / (1 + x) for x in (0, 1). -template -FixedPoint one_over_one_plus_x_for_x_in_0_1( - FixedPoint a) { - typedef FixedPoint F0; - typedef FixedPoint F2; - F0 half_denominator = RoundingHalfSum(a, F0::One()); - // Newton-Raphson division - // https://en.wikipedia.org/wiki/Division_algorithm#Newton.E2.80.93Raphson_division - // Refer to that page for the logic behind the 48/17 and 32/17 constants. - const F2 constant_48_over_17 = - GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F2, 1515870810, 48.0 / 17.0); - const F2 constant_neg_32_over_17 = - GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F2, -1010580540, -32.0 / 17.0); - F2 x = constant_48_over_17 + half_denominator * constant_neg_32_over_17; - for (int i = 0; i < 3; i++) { - F2 half_denominator_times_x = half_denominator * x; - F2 one_minus_half_denominator_times_x = - F2::One() - half_denominator_times_x; - x = x + Rescale<2>(x * one_minus_half_denominator_times_x); - } - return Rescale<0>(ExactMulByPot<-1>(x)); -} - -// Returns logistic(x) = 1 / (1 + exp(-x)) for x > 0. -template -FixedPoint logistic_on_positive_values( - FixedPoint a) { - return one_over_one_plus_x_for_x_in_0_1(exp_on_negative_values(-a)); -} - -// Returns logistic(x) = 1 / (1 + exp(-x)) for any x. -template -FixedPoint logistic(FixedPoint a) { - typedef FixedPoint InputF; - typedef FixedPoint ResultF; - tRawType mask_if_positive = MaskIfGreaterThan(a, InputF::Zero()); - tRawType mask_if_zero = MaskIfZero(a); - InputF abs_input = SelectUsingMask(mask_if_positive, a, -a); - ResultF result_if_positive = logistic_on_positive_values(abs_input); - ResultF result_if_negative = ResultF::One() - result_if_positive; - const ResultF one_half = - GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(ResultF, 1 << 30, 0.5); - return SelectUsingMask(mask_if_zero, one_half, - SelectUsingMask(mask_if_positive, result_if_positive, - result_if_negative)); -} - -} // end namespace gemmlowp - -#ifdef GEMMLOWP_NEON -#include "./fixedpoint_neon.h" -#elif defined(GEMMLOWP_AVX2) -#include "./fixedpoint_avx.h" -#elif defined(GEMMLOWP_SSE4) -#include "./fixedpoint_sse.h" -#elif defined(GEMMLOWP_MSA) -#include "./fixedpoint_msa.h" -#endif - -#endif // GEMMLOWP_INTERNAL_FIXEDPOINT_H_ diff --git a/code/lib/tfmicro/third_party/gemmlowp/fixedpoint/fixedpoint_sse.h b/code/lib/tfmicro/third_party/gemmlowp/fixedpoint/fixedpoint_sse.h deleted file mode 100644 index a1fae32d..00000000 --- a/code/lib/tfmicro/third_party/gemmlowp/fixedpoint/fixedpoint_sse.h +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// fixedpoint_SSE.h: optimized SSE specializations of the templates -// in fixedpoint.h. - -#ifndef GEMMLOWP_INTERNAL_FIXEDPOINT_SSE_H_ -#define GEMMLOWP_INTERNAL_FIXEDPOINT_SSE_H_ - -#include -#include "fixedpoint.h" - -namespace gemmlowp { - -// SSE intrinsics are not finely typed: there is a single __m128i vector -// type that does not distinguish between "int32x4" and "int16x8" use -// cases, unlike the NEON equivalents. Because we had initially focused -// on int32x4, we did not pay attention and specialized these fixedpoint -// templates directly for __m128i hardcoding the int32x4 semantics, -// not leaving room for int16x8 semantics. Amending that by adding a separate -// data type, int16x8_m128i, that wraps __m128i while being a separate -// type. -struct int16x8_m128i { - int16x8_m128i() {} - explicit int16x8_m128i(__m128i w) : v(w) {} - ~int16x8_m128i() {} - - __m128i v; -}; - -template <> -struct FixedPointRawTypeTraits<__m128i> { - typedef std::int32_t ScalarRawType; - static constexpr int kLanes = 4; -}; - -template <> -struct FixedPointRawTypeTraits { - typedef std::int16_t ScalarRawType; - static constexpr int kLanes = 8; -}; - -template <> -inline __m128i BitAnd(__m128i a, __m128i b) { - return _mm_and_si128(a, b); -} - -template <> -inline int16x8_m128i BitAnd(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_and_si128(a.v, b.v)); -} - -template <> -inline __m128i BitOr(__m128i a, __m128i b) { - return _mm_or_si128(a, b); -} - -template <> -inline int16x8_m128i BitOr(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_or_si128(a.v, b.v)); -} - -template <> -inline __m128i BitXor(__m128i a, __m128i b) { - return _mm_xor_si128(a, b); -} - -template <> -inline int16x8_m128i BitXor(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_xor_si128(a.v, b.v)); -} - -template <> -inline __m128i BitNot(__m128i a) { - return _mm_andnot_si128(a, _mm_set1_epi32(-1)); -} - -template <> -inline int16x8_m128i BitNot(int16x8_m128i a) { - return int16x8_m128i(_mm_andnot_si128(a.v, _mm_set1_epi16(-1))); -} - -template <> -inline __m128i Add(__m128i a, __m128i b) { - return _mm_add_epi32(a, b); -} - -template <> -inline int16x8_m128i Add(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_add_epi16(a.v, b.v)); -} - -template <> -inline __m128i Mul(__m128i a, __m128i b) { - return _mm_mullo_epi32(a, b); -} - -template <> -inline int16x8_m128i Mul(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_mullo_epi16(a.v, b.v)); -} - -template <> -inline __m128i Sub(__m128i a, __m128i b) { - return _mm_sub_epi32(a, b); -} - -template <> -inline int16x8_m128i Sub(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_sub_epi16(a.v, b.v)); -} - -template <> -inline __m128i Neg(__m128i a) { - return _mm_sign_epi32(a, _mm_set1_epi32(-1)); -} - -template <> -inline int16x8_m128i Neg(int16x8_m128i a) { - return int16x8_m128i(_mm_sign_epi16(a.v, _mm_set1_epi16(-1))); -} - -template <> -inline __m128i ShiftLeft(__m128i a, int offset) { - return _mm_slli_epi32(a, offset); -} - -template <> -inline int16x8_m128i ShiftLeft(int16x8_m128i a, int offset) { - return int16x8_m128i(_mm_slli_epi16(a.v, offset)); -} - -template <> -inline __m128i ShiftRight(__m128i a, int offset) { - return _mm_srai_epi32(a, offset); -} - -template <> -inline int16x8_m128i ShiftRight(int16x8_m128i a, int offset) { - return int16x8_m128i(_mm_srai_epi16(a.v, offset)); -} - -template <> -inline __m128i SelectUsingMask(__m128i if_mask, __m128i then_val, - __m128i else_val) { - // borrowed from Intel's arm_neon_sse.h header. - return _mm_or_si128(_mm_and_si128(if_mask, then_val), - _mm_andnot_si128(if_mask, else_val)); -} - -template <> -inline int16x8_m128i SelectUsingMask(int16x8_m128i if_mask, - int16x8_m128i then_val, - int16x8_m128i else_val) { - // borrowed from Intel's arm_neon_sse.h header. - return int16x8_m128i(SelectUsingMask(if_mask.v, then_val.v, else_val.v)); -} - -template <> -inline __m128i MaskIfEqual(__m128i a, __m128i b) { - return _mm_cmpeq_epi32(a, b); -} - -template <> -inline int16x8_m128i MaskIfEqual(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_cmpeq_epi16(a.v, b.v)); -} - -template <> -inline __m128i MaskIfNotEqual(__m128i a, __m128i b) { - return BitNot(MaskIfEqual(a, b)); -} - -template <> -inline int16x8_m128i MaskIfNotEqual(int16x8_m128i a, int16x8_m128i b) { - return BitNot(MaskIfEqual(a, b)); -} - -template <> -inline __m128i MaskIfZero(__m128i a) { - return MaskIfEqual(a, _mm_set1_epi32(0)); -} - -template <> -inline int16x8_m128i MaskIfZero(int16x8_m128i a) { - return MaskIfEqual(a, int16x8_m128i(_mm_set1_epi16(0))); -} - -template <> -inline __m128i MaskIfNonZero(__m128i a) { - return MaskIfNotEqual(a, _mm_set1_epi32(0)); -} - -template <> -inline int16x8_m128i MaskIfNonZero(int16x8_m128i a) { - return MaskIfNotEqual(a, int16x8_m128i(_mm_set1_epi16(0))); -} - -template <> -inline __m128i MaskIfGreaterThan(__m128i a, __m128i b) { - return _mm_cmpgt_epi32(a, b); -} - -template <> -inline int16x8_m128i MaskIfGreaterThan(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_cmpgt_epi16(a.v, b.v)); -} - -template <> -inline __m128i MaskIfLessThan(__m128i a, __m128i b) { - return _mm_cmplt_epi32(a, b); -} - -template <> -inline int16x8_m128i MaskIfLessThan(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_cmplt_epi16(a.v, b.v)); -} - -template <> -inline __m128i MaskIfGreaterThanOrEqual(__m128i a, __m128i b) { - return BitNot(MaskIfLessThan(a, b)); -} - -template <> -inline int16x8_m128i MaskIfGreaterThanOrEqual(int16x8_m128i a, - int16x8_m128i b) { - return BitNot(MaskIfLessThan(a, b)); -} - -template <> -inline __m128i MaskIfLessThanOrEqual(__m128i a, __m128i b) { - return BitNot(MaskIfGreaterThan(a, b)); -} - -template <> -inline int16x8_m128i MaskIfLessThanOrEqual(int16x8_m128i a, int16x8_m128i b) { - return BitNot(MaskIfGreaterThan(a, b)); -} - -/* Assumptions: - - All and Any are used on masks. - - masks are all_ones for true lanes, all_zeroes otherwise. -Hence, All means all 128bits set, and Any means any bit set. -*/ - -template <> -inline bool All(__m128i a) { - return _mm_testc_si128(a, a); -} - -template <> -inline bool All(int16x8_m128i a) { - return _mm_testc_si128(a.v, a.v); -} - -template <> -inline bool Any(__m128i a) { - return !_mm_testz_si128(a, a); -} - -template <> -inline bool Any(int16x8_m128i a) { - return !_mm_testz_si128(a.v, a.v); -} - -template <> -inline __m128i RoundingHalfSum(__m128i a, __m128i b) { - /* __m128i round_bit_mask, a_over_2, b_over_2, round_bit, sum; */ - /* We divide the inputs before the add to avoid the overflow and costly test - */ - /* of checking if an overflow occured on signed add */ - /* round_bit_mask = _mm_set1_epi32(1); */ - /* a_over_2 = _mm_srai_epi32(a, 1); */ - /* b_over_2 = _mm_srai_epi32(b, 1); */ - /* sum = Add(a_over_2, b_over_2); */ - /* round_bit = _mm_sign_epi32(BitAnd(BitOr(a,b), round_bit_mask), sum); */ - /* return Add(sum, round_bit); */ - - /* Other possibility detecting overflow and xor the sign if an overflow - * happened*/ - __m128i one, sign_bit_mask, sum, rounded_half_sum, overflow, result; - one = _mm_set1_epi32(1); - sign_bit_mask = _mm_set1_epi32(0x80000000); - sum = Add(a, b); - rounded_half_sum = _mm_srai_epi32(Add(sum, one), 1); - overflow = - BitAnd(BitAnd(BitXor(a, rounded_half_sum), BitXor(b, rounded_half_sum)), - sign_bit_mask); - result = BitXor(rounded_half_sum, overflow); - return result; -} - -template <> -inline int16x8_m128i RoundingHalfSum(int16x8_m128i a, int16x8_m128i b) { - // Idea: go to unsigned to use _mm_avg_epu16, - // borrowed from Intel's arm_neon_sse.h header. - __m128i constant_neg_32768 = _mm_set1_epi16(-32768); - __m128i a_unsigned = _mm_sub_epi16(a.v, constant_neg_32768); - __m128i b_unsigned = _mm_sub_epi16(b.v, constant_neg_32768); - __m128i avg_unsigned = _mm_avg_epu16(a_unsigned, b_unsigned); - __m128i avg = _mm_add_epi16(avg_unsigned, constant_neg_32768); - return int16x8_m128i(avg); -} - -template <> -inline __m128i SaturatingRoundingDoublingHighMul(__m128i a, __m128i b) { - __m128i min, saturation_mask, a0_a2, a1_a3, b0_b2, b1_b3; - __m128i a0b0_a2b2, a1b1_a3b3, a0b0_a2b2_rounded, a1b1_a3b3_rounded; - __m128i a0b0_a2b2_rounded_2x, a1b1_a3b3_rounded_2x, result; - __m128i nudge; - - // saturation only happen if a == b == INT_MIN - min = _mm_set1_epi32(std::numeric_limits::min()); - saturation_mask = BitAnd(MaskIfEqual(a, b), MaskIfEqual(a, min)); - - // a = a0 | a1 | a2 | a3 - // b = b0 | b1 | b2 | b3 - a0_a2 = a; - a1_a3 = _mm_srli_si128(a, 4); - b0_b2 = b; - b1_b3 = _mm_srli_si128(b, 4); - - a0b0_a2b2 = _mm_mul_epi32(a0_a2, b0_b2); - a1b1_a3b3 = _mm_mul_epi32(a1_a3, b1_b3); - - // do the rounding and take into account that it will be doubled - nudge = _mm_set1_epi64x(1 << 30); - a0b0_a2b2_rounded = _mm_add_epi64(a0b0_a2b2, nudge); - a1b1_a3b3_rounded = _mm_add_epi64(a1b1_a3b3, nudge); - - // do the doubling - a0b0_a2b2_rounded_2x = _mm_slli_epi64(a0b0_a2b2_rounded, 1); - a1b1_a3b3_rounded_2x = _mm_slli_epi64(a1b1_a3b3_rounded, 1); - - // get the high part of the products - result = _mm_blend_epi16(_mm_srli_si128(a0b0_a2b2_rounded_2x, 4), - a1b1_a3b3_rounded_2x, 0xcc); - - // saturate those which overflowed - return SelectUsingMask(saturation_mask, min, result); -} - -template <> -inline int16x8_m128i SaturatingRoundingDoublingHighMul(int16x8_m128i a, - int16x8_m128i b) { - // Idea: use _mm_mulhrs_epi16 then saturate with a bit-operation, - // borrowed from Intel's arm_neon_sse.h header. - __m128i result_unsaturated = _mm_mulhrs_epi16(a.v, b.v); - __m128i saturation_mask = - _mm_cmpeq_epi16(result_unsaturated, _mm_set1_epi16(0x8000)); - __m128i result = _mm_xor_si128(result_unsaturated, saturation_mask); - return int16x8_m128i(result); -} - -template <> -inline __m128i Dup<__m128i>(std::int32_t x) { - return _mm_set1_epi32(x); -} - -template <> -inline int16x8_m128i Dup(std::int16_t x) { - return int16x8_m128i(_mm_set1_epi16(x)); -} - -// So far this is only needed for int16. -template <> -inline int16x8_m128i SaturatingAdd(int16x8_m128i a, int16x8_m128i b) { - return int16x8_m128i(_mm_adds_epi16(a.v, b.v)); -} - -} // end namespace gemmlowp - -#endif // GEMMLOWP_INTERNAL_FIXEDPOINT_SSE_H_ diff --git a/code/lib/tfmicro/third_party/gemmlowp/internal/detect_platform.h b/code/lib/tfmicro/third_party/gemmlowp/internal/detect_platform.h deleted file mode 100644 index 6f06d19f..00000000 --- a/code/lib/tfmicro/third_party/gemmlowp/internal/detect_platform.h +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2018 The Gemmlowp Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// detect_platform.h: Sets up macros that control architecture-specific -// features of gemmlowp's implementation. - -#ifndef GEMMLOWP_INTERNAL_DETECT_PLATFORM_H_ -#define GEMMLOWP_INTERNAL_DETECT_PLATFORM_H_ - -// Our inline assembly path assume GCC/Clang syntax. -// Native Client doesn't seem to support inline assembly(?). -#if defined(__GNUC__) && !defined(__native_client__) -#define GEMMLOWP_ALLOW_INLINE_ASM -#endif - -// Define macro statement that avoids inlining for GCC. -// For non-GCC, define as empty macro. -#if defined(__GNUC__) -#define GEMMLOWP_NOINLINE __attribute__((noinline)) -#else -#define GEMMLOWP_NOINLINE -#endif - -// Detect ARM, 32-bit or 64-bit -#ifdef __arm__ -#define GEMMLOWP_ARM_32 -#endif - -#ifdef __aarch64__ -#define GEMMLOWP_ARM_64 -#endif - -#if defined(GEMMLOWP_ARM_32) || defined(GEMMLOWP_ARM_64) -#define GEMMLOWP_ARM -#endif - -// Detect MIPS, 32-bit or 64-bit -#if defined(__mips) && !defined(__LP64__) -#define GEMMLOWP_MIPS_32 -#endif - -#if defined(__mips) && defined(__LP64__) -#define GEMMLOWP_MIPS_64 -#endif - -#if defined(GEMMLOWP_MIPS_32) || defined(GEMMLOWP_MIPS_64) -#define GEMMLOWP_MIPS -#endif - -// Detect x86, 32-bit or 64-bit -#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || defined(__i386) -#define GEMMLOWP_X86_32 -#endif - -#if defined(__x86_64__) || defined(_M_X64) || defined(__amd64) -#define GEMMLOWP_X86_64 -#endif - -#if defined(GEMMLOWP_X86_32) || defined(GEMMLOWP_X86_64) -#define GEMMLOWP_X86 -#endif - -// Some of our optimized paths use inline assembly and for -// now we don't bother enabling some other optimized paths using intrinddics -// where we can't use inline assembly paths. -#ifdef GEMMLOWP_ALLOW_INLINE_ASM - -// Detect NEON. It's important to check for both tokens. -#if (defined __ARM_NEON) || (defined __ARM_NEON__) -#define GEMMLOWP_NEON -#endif - -// Convenience NEON tokens for 32-bit or 64-bit -#if defined(GEMMLOWP_NEON) && defined(GEMMLOWP_ARM_32) -#define GEMMLOWP_NEON_32 -#endif - -#if defined(GEMMLOWP_NEON) && defined(GEMMLOWP_ARM_64) -#define GEMMLOWP_NEON_64 -#endif - -// Detect MIPS MSA. -// Limit MSA optimizations to little-endian CPUs for now. -// TODO: Perhaps, eventually support MSA optimizations on big-endian CPUs? -#if defined(GEMMLOWP_MIPS) && (__mips_isa_rev >= 5) && defined(__mips_msa) && \ - defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -#define GEMMLOWP_MSA -#endif - -// Convenience MIPS MSA tokens for 32-bit or 64-bit. -#if defined(GEMMLOWP_MSA) && defined(GEMMLOWP_MIPS_32) -#define GEMMLOWP_MSA_32 -#endif - -#if defined(GEMMLOWP_MSA) && defined(GEMMLOWP_MIPS_64) -#define GEMMLOWP_MSA_64 -#endif - -// compiler define for AVX2 -D GEMMLOWP_ENABLE_AVX2 -// Detect AVX2 -#if defined(__AVX2__) && defined(GEMMLOWP_ENABLE_AVX2) -#define GEMMLOWP_AVX2 -// Detect SSE4. -// MSVC does not have __SSE4_1__ macro, but will enable SSE4 -// when AVX is turned on. -#elif defined(__SSE4_1__) || (defined(_MSC_VER) && defined(__AVX__)) -#define GEMMLOWP_SSE4 -// Detect SSE3. -#elif defined(__SSE3__) -#define GEMMLOWP_SSE3 -#endif - -// Convenience SSE4 tokens for 32-bit or 64-bit -#if defined(GEMMLOWP_SSE4) && defined(GEMMLOWP_X86_32) && \ - !defined(GEMMLOWP_DISABLE_SSE4) -#define GEMMLOWP_SSE4_32 -#endif - -#if defined(GEMMLOWP_SSE3) && defined(GEMMLOWP_X86_32) -#define GEMMLOWP_SSE3_32 -#endif - -#if defined(GEMMLOWP_SSE4) && defined(GEMMLOWP_X86_64) && \ - !defined(GEMMLOWP_DISABLE_SSE4) -#define GEMMLOWP_SSE4_64 -#endif - -#if defined(GEMMLOWP_SSE3) && defined(GEMMLOWP_X86_64) -#define GEMMLOWP_SSE3_64 -#endif - -#if defined(GEMMLOWP_AVX2) && defined(GEMMLOWP_X86_64) -#define GEMMLOWP_AVX2_64 -#endif - -#if defined(__has_feature) -#if __has_feature(memory_sanitizer) -#include -#define GEMMLOWP_MARK_MEMORY_AS_INITIALIZED __msan_unpoison -#elif __has_feature(address_sanitizer) -#include -#define GEMMLOWP_MARK_MEMORY_AS_INITIALIZED __asan_unpoison_memory_region -#endif -#endif - -#endif // GEMMLOWP_ALLOW_INLINE_ASM - -// Detect Android. Don't conflate with ARM - we care about tuning -// for non-ARM Android devices too. This can be used in conjunction -// with x86 to tune differently for mobile x86 CPUs (Atom) vs. desktop x86 CPUs. -#if defined(__ANDROID__) || defined(ANDROID) -#define GEMMLOWP_ANDROID -#endif - -#endif // GEMMLOWP_INTERNAL_DETECT_PLATFORM_H_ diff --git a/code/lib/tfmicro/third_party/kissfft/COPYING b/code/lib/tfmicro/third_party/kissfft/COPYING deleted file mode 100644 index 2fc6685a..00000000 --- a/code/lib/tfmicro/third_party/kissfft/COPYING +++ /dev/null @@ -1,11 +0,0 @@ -Copyright (c) 2003-2010 Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/code/lib/tfmicro/third_party/kissfft/_kiss_fft_guts.h b/code/lib/tfmicro/third_party/kissfft/_kiss_fft_guts.h deleted file mode 100644 index ba661444..00000000 --- a/code/lib/tfmicro/third_party/kissfft/_kiss_fft_guts.h +++ /dev/null @@ -1,164 +0,0 @@ -/* -Copyright (c) 2003-2010, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/* kiss_fft.h - defines kiss_fft_scalar as either short or a float type - and defines - typedef struct { kiss_fft_scalar r; kiss_fft_scalar i; }kiss_fft_cpx; */ -#include "kiss_fft.h" -#include - -#define MAXFACTORS 32 -/* e.g. an fft of length 128 has 4 factors - as far as kissfft is concerned - 4*4*4*2 - */ - -struct kiss_fft_state{ - int nfft; - int inverse; - int factors[2*MAXFACTORS]; - kiss_fft_cpx twiddles[1]; -}; - -/* - Explanation of macros dealing with complex math: - - C_MUL(m,a,b) : m = a*b - C_FIXDIV( c , div ) : if a fixed point impl., c /= div. noop otherwise - C_SUB( res, a,b) : res = a - b - C_SUBFROM( res , a) : res -= a - C_ADDTO( res , a) : res += a - * */ -#ifdef FIXED_POINT -#if (FIXED_POINT==32) -# define FRACBITS 31 -# define SAMPPROD int64_t -#define SAMP_MAX 2147483647 -#else -# define FRACBITS 15 -# define SAMPPROD int32_t -#define SAMP_MAX 32767 -#endif - -#define SAMP_MIN -SAMP_MAX - -#if defined(CHECK_OVERFLOW) -# define CHECK_OVERFLOW_OP(a,op,b) \ - if ( (SAMPPROD)(a) op (SAMPPROD)(b) > SAMP_MAX || (SAMPPROD)(a) op (SAMPPROD)(b) < SAMP_MIN ) { \ - fprintf(stderr,"WARNING:overflow @ " __FILE__ "(%d): (%d " #op" %d) = %ld\n",__LINE__,(a),(b),(SAMPPROD)(a) op (SAMPPROD)(b) ); } -#endif - - -# define smul(a,b) ( (SAMPPROD)(a)*(b) ) -# define sround( x ) (kiss_fft_scalar)( ( (x) + (1<<(FRACBITS-1)) ) >> FRACBITS ) - -# define S_MUL(a,b) sround( smul(a,b) ) - -# define C_MUL(m,a,b) \ - do{ (m).r = sround( smul((a).r,(b).r) - smul((a).i,(b).i) ); \ - (m).i = sround( smul((a).r,(b).i) + smul((a).i,(b).r) ); }while(0) - -# define DIVSCALAR(x,k) \ - (x) = sround( smul( x, SAMP_MAX/k ) ) - -# define C_FIXDIV(c,div) \ - do { DIVSCALAR( (c).r , div); \ - DIVSCALAR( (c).i , div); }while (0) - -# define C_MULBYSCALAR( c, s ) \ - do{ (c).r = sround( smul( (c).r , s ) ) ;\ - (c).i = sround( smul( (c).i , s ) ) ; }while(0) - -#else /* not FIXED_POINT*/ - -# define S_MUL(a,b) ( (a)*(b) ) -#define C_MUL(m,a,b) \ - do{ (m).r = (a).r*(b).r - (a).i*(b).i;\ - (m).i = (a).r*(b).i + (a).i*(b).r; }while(0) -# define C_FIXDIV(c,div) /* NOOP */ -# define C_MULBYSCALAR( c, s ) \ - do{ (c).r *= (s);\ - (c).i *= (s); }while(0) -#endif - -#ifndef CHECK_OVERFLOW_OP -# define CHECK_OVERFLOW_OP(a,op,b) /* noop */ -#endif - -#define C_ADD( res, a,b)\ - do { \ - CHECK_OVERFLOW_OP((a).r,+,(b).r)\ - CHECK_OVERFLOW_OP((a).i,+,(b).i)\ - (res).r=(a).r+(b).r; (res).i=(a).i+(b).i; \ - }while(0) -#define C_SUB( res, a,b)\ - do { \ - CHECK_OVERFLOW_OP((a).r,-,(b).r)\ - CHECK_OVERFLOW_OP((a).i,-,(b).i)\ - (res).r=(a).r-(b).r; (res).i=(a).i-(b).i; \ - }while(0) -#define C_ADDTO( res , a)\ - do { \ - CHECK_OVERFLOW_OP((res).r,+,(a).r)\ - CHECK_OVERFLOW_OP((res).i,+,(a).i)\ - (res).r += (a).r; (res).i += (a).i;\ - }while(0) - -#define C_SUBFROM( res , a)\ - do {\ - CHECK_OVERFLOW_OP((res).r,-,(a).r)\ - CHECK_OVERFLOW_OP((res).i,-,(a).i)\ - (res).r -= (a).r; (res).i -= (a).i; \ - }while(0) - - -#ifdef FIXED_POINT -# define KISS_FFT_COS(phase) floor(.5+SAMP_MAX * cos (phase)) -# define KISS_FFT_SIN(phase) floor(.5+SAMP_MAX * sin (phase)) -# define HALF_OF(x) ((x)>>1) -#elif defined(USE_SIMD) -# define KISS_FFT_COS(phase) _mm_set1_ps( cos(phase) ) -# define KISS_FFT_SIN(phase) _mm_set1_ps( sin(phase) ) -# define HALF_OF(x) ((x)*_mm_set1_ps(.5)) -#else -# define KISS_FFT_COS(phase) (kiss_fft_scalar) cos(phase) -# define KISS_FFT_SIN(phase) (kiss_fft_scalar) sin(phase) -# define HALF_OF(x) ((x)*.5) -#endif - -#define kf_cexp(x,phase) \ - do{ \ - (x)->r = KISS_FFT_COS(phase);\ - (x)->i = KISS_FFT_SIN(phase);\ - }while(0) - - -/* a debugging function */ -#define pcpx(c)\ - fprintf(stderr,"%g + %gi\n",(double)((c)->r),(double)((c)->i) ) - - -#ifdef KISS_FFT_USE_ALLOCA -// define this to allow use of alloca instead of malloc for temporary buffers -// Temporary buffers are used in two case: -// 1. FFT sizes that have "bad" factors. i.e. not 2,3 and 5 -// 2. "in-place" FFTs. Notice the quotes, since kissfft does not really do an in-place transform. -#include -#define KISS_FFT_TMP_ALLOC(nbytes) alloca(nbytes) -#define KISS_FFT_TMP_FREE(ptr) -#else -#define KISS_FFT_TMP_ALLOC(nbytes) KISS_FFT_MALLOC(nbytes) -#define KISS_FFT_TMP_FREE(ptr) KISS_FFT_FREE(ptr) -#endif diff --git a/code/lib/tfmicro/third_party/kissfft/kiss_fft.h b/code/lib/tfmicro/third_party/kissfft/kiss_fft.h deleted file mode 100644 index c34ea5ee..00000000 --- a/code/lib/tfmicro/third_party/kissfft/kiss_fft.h +++ /dev/null @@ -1,131 +0,0 @@ -#ifndef KISS_FFT_H -#define KISS_FFT_H - -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* - ATTENTION! - If you would like a : - -- a utility that will handle the caching of fft objects - -- real-only (no imaginary time component ) FFT - -- a multi-dimensional FFT - -- a command-line utility to perform ffts - -- a command-line utility to perform fast-convolution filtering - - Then see kfc.h kiss_fftr.h kiss_fftnd.h fftutil.c kiss_fastfir.c - in the tools/ directory. -*/ - -#ifdef USE_SIMD -# include -# define kiss_fft_scalar __m128 -#define KISS_FFT_MALLOC(nbytes) _mm_malloc(nbytes,16) -#define KISS_FFT_FREE _mm_free -#else -#define KISS_FFT_MALLOC(X) (void*)(0) /* Patched. */ -#define KISS_FFT_FREE(X) /* Patched. */ -#endif - - -// Patched automatically by download_dependencies.sh so default is 16 bit. -#ifndef FIXED_POINT -#define FIXED_POINT (16) -#endif -// End patch. - -#ifdef FIXED_POINT -#include /* Patched. */ -#include -# if (FIXED_POINT == 32) -# define kiss_fft_scalar int32_t -# else -# define kiss_fft_scalar int16_t -# endif -#else -# ifndef kiss_fft_scalar -/* default is float */ -# define kiss_fft_scalar float -# endif -#endif - -typedef struct { - kiss_fft_scalar r; - kiss_fft_scalar i; -}kiss_fft_cpx; - -typedef struct kiss_fft_state* kiss_fft_cfg; - -/* - * kiss_fft_alloc - * - * Initialize a FFT (or IFFT) algorithm's cfg/state buffer. - * - * typical usage: kiss_fft_cfg mycfg=kiss_fft_alloc(1024,0,NULL,NULL); - * - * The return value from fft_alloc is a cfg buffer used internally - * by the fft routine or NULL. - * - * If lenmem is NULL, then kiss_fft_alloc will allocate a cfg buffer using malloc. - * The returned value should be free()d when done to avoid memory leaks. - * - * The state can be placed in a user supplied buffer 'mem': - * If lenmem is not NULL and mem is not NULL and *lenmem is large enough, - * then the function places the cfg in mem and the size used in *lenmem - * and returns mem. - * - * If lenmem is not NULL and ( mem is NULL or *lenmem is not large enough), - * then the function returns NULL and places the minimum cfg - * buffer size in *lenmem. - * */ - -kiss_fft_cfg kiss_fft_alloc(int nfft,int inverse_fft,void * mem,size_t * lenmem); - -/* - * kiss_fft(cfg,in_out_buf) - * - * Perform an FFT on a complex input buffer. - * for a forward FFT, - * fin should be f[0] , f[1] , ... ,f[nfft-1] - * fout will be F[0] , F[1] , ... ,F[nfft-1] - * Note that each element is complex and can be accessed like - f[k].r and f[k].i - * */ -void kiss_fft(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout); - -/* - A more generic version of the above function. It reads its input from every Nth sample. - * */ -void kiss_fft_stride(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout,int fin_stride); - -/* If kiss_fft_alloc allocated a buffer, it is one contiguous - buffer and can be simply free()d when no longer needed*/ -#define kiss_fft_free free - -/* - Cleans up some memory that gets managed internally. Not necessary to call, but it might clean up - your compiler output to call this before you exit. -*/ -void kiss_fft_cleanup(void); - - -/* - * Returns the smallest integer k, such that k>=n and k has only "fast" factors (2,3,5) - */ -int kiss_fft_next_fast_size(int n); - -/* for real ffts, we need an even size */ -#define kiss_fftr_next_fast_size_real(n) \ - (kiss_fft_next_fast_size( ((n)+1)>>1)<<1) - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/code/lib/tfmicro/third_party/kissfft/tools/kiss_fftr.h b/code/lib/tfmicro/third_party/kissfft/tools/kiss_fftr.h deleted file mode 100644 index 72e5a577..00000000 --- a/code/lib/tfmicro/third_party/kissfft/tools/kiss_fftr.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef KISS_FTR_H -#define KISS_FTR_H - -#include "kiss_fft.h" -#ifdef __cplusplus -extern "C" { -#endif - - -/* - - Real optimized version can save about 45% cpu time vs. complex fft of a real seq. - - - - */ - -typedef struct kiss_fftr_state *kiss_fftr_cfg; - - -kiss_fftr_cfg kiss_fftr_alloc(int nfft,int inverse_fft,void * mem, size_t * lenmem); -/* - nfft must be even - - If you don't care to allocate space, use mem = lenmem = NULL -*/ - - -void kiss_fftr(kiss_fftr_cfg cfg,const kiss_fft_scalar *timedata,kiss_fft_cpx *freqdata); -/* - input timedata has nfft scalar points - output freqdata has nfft/2+1 complex points -*/ - -void kiss_fftri(kiss_fftr_cfg cfg,const kiss_fft_cpx *freqdata,kiss_fft_scalar *timedata); -/* - input freqdata has nfft/2+1 complex points - output timedata has nfft scalar points -*/ - -#define kiss_fftr_free free - -#ifdef __cplusplus -} -#endif -#endif diff --git a/code/lib/tfmicro/third_party/ruy/ruy/profiler/instrumentation.h b/code/lib/tfmicro/third_party/ruy/ruy/profiler/instrumentation.h deleted file mode 100644 index c4df1e68..00000000 --- a/code/lib/tfmicro/third_party/ruy/ruy/profiler/instrumentation.h +++ /dev/null @@ -1,203 +0,0 @@ -/* Copyright 2020 Google LLC. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef RUY_RUY_PROFILER_INSTRUMENTATION_H_ -#define RUY_RUY_PROFILER_INSTRUMENTATION_H_ - -#ifdef RUY_PROFILER -#include -#include -#include -#endif - -namespace ruy { -namespace profiler { - -#ifdef RUY_PROFILER - -// A label is how a code scope is annotated to appear in profiles. -// The stacks that are sampled by the profiler are stacks of such labels. -// A label consists of a literal string, plus optional integer arguments. -class Label { - public: - Label() {} - template - explicit Label(Args... args) { - Set(args...); - } - void Set(const char* format) { - format_ = format; - args_count_ = 0; - } - template - void Set(const char* format, Args... args) { - format_ = format; - args_count_ = sizeof...(args); - SetArgs(0, args...); - } - - void operator=(const Label& other); - - bool operator==(const Label& other) const; - - std::string Formatted() const; - const char* format() const { return format_; } - - private: - void SetArgs(int position, int arg0) { args_[position] = arg0; } - - template - void SetArgs(int position, int arg0, Args... args) { - SetArgs(position, arg0); - SetArgs(position + 1, args...); - } - - static constexpr int kMaxArgs = 4; - const char* format_ = nullptr; - int args_count_ = 0; - int args_[kMaxArgs]; -}; - -namespace detail { - -// Forward-declaration, see class ThreadStack below. -class ThreadStack; - -bool& GlobalIsProfilerRunning(); - -// Returns the global vector of pointers to all stacks, there being one stack -// per thread executing instrumented code. -std::vector* GlobalAllThreadStacks(); - -// Returns the mutex to be locked around any access to GlobalAllThreadStacks(). -std::mutex* GlobalsMutex(); - -// Returns the thread-local stack, specific to the current thread. -ThreadStack* ThreadLocalThreadStack(); - -// This 'stack' is what may be more appropriately called a 'pseudostack': -// It contains Label entries that are 'manually' entered by instrumentation -// code. It's unrelated to real call stacks. -struct Stack { - std::uint32_t id = 0; - static constexpr int kMaxSize = 64; - int size = 0; - Label labels[kMaxSize]; -}; - -// Returns the buffer byte size required by CopyToSample. -int GetBufferSize(const Stack& stack); - -// Copies this Stack into a byte buffer, called a 'sample'. -void CopyToBuffer(const Stack& stack, char* dst); - -// Populates this Stack from an existing sample buffer, typically -// produced by CopyToSample. -void ReadFromBuffer(const char* src, Stack* stack); - -// ThreadStack is meant to be used as a thread-local singleton, assigning to -// each thread a Stack object holding its pseudo-stack of profile labels, -// plus a mutex allowing to synchronize accesses to this pseudo-stack between -// this thread and a possible profiler thread sampling it. -class ThreadStack { - public: - ThreadStack(); - ~ThreadStack(); - - const Stack& stack() const { return stack_; } - - // Returns the mutex to lock around any access to this stack. Each stack is - // accessed by potentially two threads: the thread that it belongs to - // (which calls Push and Pop) and the profiler thread during profiling - // (which calls CopyToSample). - std::mutex& Mutex() const { return mutex_; } - - // Pushes a new label on the top of this Stack. - template - void Push(Args... args) { - // This mutex locking is needed to guard against race conditions as both - // the current thread and the profiler thread may be concurrently accessing - // this stack. In addition to that, this mutex locking also serves the other - // purpose of acting as a barrier (of compiler code reordering, of runtime - // CPU instruction reordering, and of memory access reordering), which - // gives a measure of correctness to this profiler. The downside is some - // latency. As this lock will be uncontended most of the times, the cost - // should be roughly that of an sequentially-consistent atomic access, - // comparable to an access to the level of CPU data cache that is shared - // among all cores, typically 60 cycles on current ARM CPUs, plus side - // effects from barrier instructions. - std::lock_guard lock(mutex_); - // Avoid overrunning the stack, even in 'release' builds. This profiling - // instrumentation code should not ship in release builds anyway, the - // overhead of this check is negligible, and overrunning a stack array would - // be bad. - if (stack_.size >= Stack::kMaxSize) { - abort(); - } - stack_.labels[stack_.size++].Set(args...); - } - - // Pops the top-most label from this Stack. - void Pop() { - // See the comment in Push about this lock. While it would be tempting to - // try to remove this lock and just atomically decrement size_ with a - // store-release, that would not necessarily be a substitute for all of the - // purposes that this lock serves, or if it was done carefully to serve all - // of the same purposes, then that wouldn't be faster than this (mostly - // uncontended) lock. - std::lock_guard lock(mutex_); - stack_.size--; - } - - private: - mutable std::mutex mutex_; - Stack stack_; -}; - -} // namespace detail - -// RAII user-facing way to construct Labels associated with their life scope -// and get them pushed to / popped from the current thread stack. -class ScopeLabel { - public: - template - ScopeLabel(Args... args) : thread_stack_(detail::ThreadLocalThreadStack()) { - thread_stack_->Push(args...); - } - - ~ScopeLabel() { thread_stack_->Pop(); } - - private: - detail::ThreadStack* thread_stack_; -}; - -#else // no RUY_PROFILER - -class ScopeLabel { - public: - template - explicit ScopeLabel(Args...) {} - - // This destructor is needed to consistently silence clang's -Wunused-variable - // which seems to trigger semi-randomly. - ~ScopeLabel() {} -}; - -#endif - -} // namespace profiler -} // namespace ruy - -#endif // RUY_RUY_PROFILER_INSTRUMENTATION_H_ diff --git a/code/src/server_main.h b/code/src/server_main.h index 83471970..16a851ed 100644 --- a/code/src/server_main.h +++ b/code/src/server_main.h @@ -14,7 +14,7 @@ #include -static const char *TAG = "example"; +static const char *TAG = "server-main"; extern httpd_handle_t server; diff --git a/code/src/server_tflite.cpp b/code/src/server_tflite.cpp index f5e2d5e0..b579ac59 100644 --- a/code/src/server_tflite.cpp +++ b/code/src/server_tflite.cpp @@ -58,7 +58,8 @@ void doInit(void) bool doflow(void) { - std::string zw_time = gettimestring("%Y%m%d-%H%M%S"); + + std::string zw_time = gettimestring(LOGFILE_TIME_FORMAT); printf("doflow - start %s\n", zw_time.c_str()); flowisrunning = true; tfliteflow.doFlow(zw_time); @@ -433,6 +434,8 @@ void task_autodoFlow(void *pvParameter) printf("Autoflow: doFLow wird gestartet\n"); flowisrunning = true; doflow(); + printf("Remove older log files\n"); + LogFile.RemoveOld(); } LogFile.WriteToFile("task_autodoFlow - round done"); diff --git a/code/src/version.cpp b/code/src/version.cpp index d1393fa2..8911e364 100644 --- a/code/src/version.cpp +++ b/code/src/version.cpp @@ -1,4 +1,4 @@ -const char* GIT_REV="2ab2f07"; +const char* GIT_REV="707472b"; const char* GIT_TAG=""; -const char* GIT_BRANCH="master"; -const char* BUILD_TIME="2020-11-03 22:12"; \ No newline at end of file +const char* GIT_BRANCH="rolling"; +const char* BUILD_TIME="2020-11-15 11:59"; \ No newline at end of file diff --git a/code/src/version.h b/code/src/version.h index 6570e6f9..7d39a034 100644 --- a/code/src/version.h +++ b/code/src/version.h @@ -13,7 +13,7 @@ extern "C" #include "Helper.h" #include -const char* GIT_BASE_BRANCH = "master - v3.1.0 - 2020-10-26"; +const char* GIT_BASE_BRANCH = "master - v4.0.0 - 2020-11-15"; const char* git_base_branch(void) diff --git a/code/src/zip._c b/code/src/zip._c deleted file mode 100644 index e69de29b..00000000 diff --git a/code/version.cpp b/code/version.cpp index d1393fa2..8911e364 100644 --- a/code/version.cpp +++ b/code/version.cpp @@ -1,4 +1,4 @@ -const char* GIT_REV="2ab2f07"; +const char* GIT_REV="707472b"; const char* GIT_TAG=""; -const char* GIT_BRANCH="master"; -const char* BUILD_TIME="2020-11-03 22:12"; \ No newline at end of file +const char* GIT_BRANCH="rolling"; +const char* BUILD_TIME="2020-11-15 11:59"; \ No newline at end of file diff --git a/firmware/bootloader.bin b/firmware/bootloader.bin index edb2308e..b41cfc0a 100644 Binary files a/firmware/bootloader.bin and b/firmware/bootloader.bin differ diff --git a/firmware/firmware.bin b/firmware/firmware.bin index 1f5bbce8..8e69fef0 100644 Binary files a/firmware/firmware.bin and b/firmware/firmware.bin differ diff --git a/firmware/html.zip b/firmware/html.zip index 6c5b3e56..914d6404 100644 Binary files a/firmware/html.zip and b/firmware/html.zip differ diff --git a/sd-card/config/config.ini b/sd-card/config/config.ini index 64219ec0..df9ba8a8 100644 --- a/sd-card/config/config.ini +++ b/sd-card/config/config.ini @@ -1,5 +1,6 @@ [MakeImage] ;LogImageLocation = /log/source +;LogfileRetentionInDays = 15 WaitBeforeTakingPicture=5 ImageQuality = 5 ImageSize = VGA @@ -15,6 +16,7 @@ SearchFieldY = 20 [Digits] Model=/config/dig0650s3.tflite LogImageLocation = /log/digit +LogfileRetentionInDays = 15 ModelInputSize 20, 32 digit1, 306, 120, 37, 67 digit2, 355, 120, 37, 67 @@ -23,6 +25,7 @@ digit3, 404, 120, 37, 67 [Analog] Model=/config/ana0630s2.tflite LogImageLocation = /log/analog +LogfileRetentionInDays = 15 ModelInputSize 32, 32 analog1, 444, 225, 92, 92 analog2, 391, 329, 92, 92 @@ -51,5 +54,7 @@ Intervall = 4.85 [Debug] Logfile = False +; Number of days before a log file is deleted. 0 = disabled. 10 is default value (if not defined) +;LogfileRetentionInDays = 10 [Ende] \ No newline at end of file diff --git a/sd-card/config/dig0650s3.tflite b/sd-card/config/dig0650s3.tflite index 56ac6a92..1c81acaf 100644 Binary files a/sd-card/config/dig0650s3.tflite and b/sd-card/config/dig0650s3.tflite differ diff --git a/sd-card/html/index.html b/sd-card/html/index.html index 36e2fa78..78e55a9d 100644 --- a/sd-card/html/index.html +++ b/sd-card/html/index.html @@ -91,7 +91,7 @@ li.dropdown { System
OTA Update - Log Viewer + Log Viewer Reboot Info
diff --git a/sd-card/html/version.txt b/sd-card/html/version.txt index 9084fa2f..26aaba0e 100644 --- a/sd-card/html/version.txt +++ b/sd-card/html/version.txt @@ -1 +1 @@ -1.1.0 +1.2.0