diff --git a/README.md b/README.md index deb6f8a8..c6561a19 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,16 @@ In other cases you can contact the developer via email: - +#include "esp_nn_defs.h" /************************** Basic math functions ****************************/ /** @@ -81,28 +80,15 @@ void esp_nn_mul_elementwise_s8_ansi(const int8_t *input1_data, * optimization notes: Though input_offset is int32 type, * offset values are contained in 8 bits [-128, 127] */ -void esp_nn_depthwise_conv_s8_ansi(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, - const uint16_t ch_mult, +void esp_nn_depthwise_conv_s8_ansi(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, + const data_dims_t *output_dims, int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max); + const dw_conv_params_t *conv_params, + const quant_data_t *quant_data); /** * @brief 2d-convolution channelwise @@ -112,43 +98,26 @@ void esp_nn_depthwise_conv_s8_ansi(const int8_t *input_data, * inputs type: int8_t, output: int8_t * input offsets: although int32_t, they are contained in 8 bits [-128, 127] */ -void esp_nn_conv_s8_ansi(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, +void esp_nn_conv_s8_ansi(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, + const data_dims_t *output_dims, int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const uint16_t out_channels, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max); + const conv_params_t *conv_params, + const quant_data_t *quant_data); -int esp_nn_get_conv_scratch_size_ansi(const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_ch, - const uint16_t out_ch, - const uint16_t filter_wd, - const uint16_t filter_ht); +int esp_nn_get_conv_scratch_size_ansi(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const conv_params_t *conv_params); void esp_nn_set_conv_scratch_buf_ansi(const void *buf); -int esp_nn_get_depthwise_conv_scratch_size_ansi(const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const uint16_t ch_mult, - const uint16_t filter_wd, - const uint16_t filter_ht); +int esp_nn_get_depthwise_conv_scratch_size_ansi(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const dw_conv_params_t *conv_params); void esp_nn_set_depthwise_conv_scratch_buf_ansi(const void *buf); /************************** Activation functions *****************************/ @@ -252,9 +221,6 @@ int32_t esp_nn_get_softmax_scratch_size_opt(const int32_t width, const int32_t h */ void esp_nn_set_softmax_scratch_buf_ansi(void *buffer); -/* ANSI C function to be hooked up when optimised version needed */ -void esp_nn_set_softmax_scratch_buf_opt(void *buffer); - /** * @brief reference softmax function * @@ -268,6 +234,66 @@ void esp_nn_softmax_s8_ansi(const int8_t *input_data, const int32_t diff_min, int8_t *output_data); + +//////////////////////////// Generic optimisations ///////////////////////////// + +/************************** Convolution functions *****************************/ + +/** + * @brief 2d-convolution channelwise optimized version + * + * @note operation: result += (input + offset) * filter + * + * inputs type: int8_t, output: int8_t + * input offsets: although int32_t, they are contained in 8 bits [-128, 127] + */ +void esp_nn_conv_s8_opt(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, + const int8_t *filter_data, + const int32_t *bias, + const data_dims_t *output_dims, + int8_t *out_data, + const conv_params_t *conv_params, + const quant_data_t *quant_data); + +/** + * @brief depthwise convolution per channel optimized version + * + * @note inputs type: int8_t, output: int8_t + * Version used in tflite is per channel. + * This version follows the same footsprints. + * Meaning, it has per out_channel shift and multiplier for + * requantization + * + * optimization notes: Though input_offset is int32 type, + * offset values are contained in 8 bits [-128, 127] + */ +void esp_nn_depthwise_conv_s8_opt(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, + const int8_t *filter_data, + const int32_t *bias, + const data_dims_t *output_dims, + int8_t *out_data, + const dw_conv_params_t *conv_params, + const quant_data_t *quant_data); + +int esp_nn_get_conv_scratch_size_opt(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const conv_params_t *conv_params); +void esp_nn_set_conv_scratch_buf_opt(const void *buf); + +int esp_nn_get_depthwise_conv_scratch_size_opt(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const dw_conv_params_t *conv_params); +void esp_nn_set_depthwise_conv_scratch_buf_opt(const void *buf); + +/* ANSI C function to be hooked up when optimised version needed */ +void esp_nn_set_softmax_scratch_buf_opt(void *buffer); + /** * @brief optimised version of softmax function * diff --git a/code/components/esp-nn/include/esp_nn_defs.h b/code/components/esp-nn/include/esp_nn_defs.h new file mode 100644 index 00000000..756d8e6f --- /dev/null +++ b/code/components/esp-nn/include/esp_nn_defs.h @@ -0,0 +1,83 @@ +// Copyright 2022 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +/** + * @brief structure to club data dims + * this structure can be used for input, output and filter + */ +typedef struct data_dims { + int32_t width; + int32_t height; + int32_t channels; + + int32_t extra; // can be used as batch or any other param +} data_dims_t; + +/** + * @brief 2d data structure (width, height) + * + */ +typedef struct data_2d { + int32_t width; + int32_t height; +} data_2d_t; + +/** + * @brief min/max activation + */ +typedef struct act_params { + int32_t min; + int32_t max; +} act_params_t; + +/** + * @brief per channel quant data + * + * @note number of shift and mult elements are equal to output channels + */ +typedef struct quant_data { + int32_t *shift; + int32_t *mult; +} quant_data_t; + +/** + * @brief params specific to convolution 2d + * + */ +typedef struct conv_params { + int32_t in_offset; + int32_t out_offset; + data_2d_t stride; + data_2d_t padding; + data_2d_t dilation; + act_params_t activation; +} conv_params_t; + +/** + * @brief params specific to depthwise convolution 2d + * + */ +typedef struct dw_conv_params { + int32_t in_offset; + int32_t out_offset; + int32_t ch_mult; // channel multiplier. (in_ch * ch_mult = out_ch) + data_2d_t stride; + data_2d_t padding; + data_2d_t dilation; + act_params_t activation; +} dw_conv_params_t; diff --git a/code/components/esp-nn/include/esp_nn_esp32s3.h b/code/components/esp-nn/include/esp_nn_esp32s3.h index 58b544e4..0f52c943 100644 --- a/code/components/esp-nn/include/esp_nn_esp32s3.h +++ b/code/components/esp-nn/include/esp_nn_esp32s3.h @@ -19,7 +19,7 @@ #pragma once -#include +#include "esp_nn_defs.h" #include "esp_nn_ansi_headers.h" /************************** Basic math functions *****************************/ @@ -85,28 +85,15 @@ void esp_nn_mul_elementwise_s8_esp32s3(const int8_t *input1_data, * optimization notes: Though input_offset is int32 type, * offset values are contained in 8 bits [-128, 127] */ -void esp_nn_depthwise_conv_s8_esp32s3(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, - const uint16_t ch_mult, +void esp_nn_depthwise_conv_s8_esp32s3(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, - int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max); + const data_dims_t *output_dims, + int8_t *output_data, + const dw_conv_params_t *conv_params, + const quant_data_t *quant_data); /** * @brief 2d - convolution channelwise @@ -116,43 +103,26 @@ void esp_nn_depthwise_conv_s8_esp32s3(const int8_t *input_data, * inputs type: int8_t, output: int8_t * input offsets: although int32_t, they are contained in 8 bits [-128, 127] */ -void esp_nn_conv_s8_esp32s3(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, +void esp_nn_conv_s8_esp32s3(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, - int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const uint16_t out_channels, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max); + const data_dims_t *output_dims, + int8_t *output_data, + const conv_params_t *conv_params, + const quant_data_t *quant_data); -int esp_nn_get_conv_scratch_size_esp32s3(const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_ch, - const uint16_t out_ch, - const uint16_t filter_wd, - const uint16_t filter_ht); +int esp_nn_get_conv_scratch_size_esp32s3(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const conv_params_t *conv_params); void esp_nn_set_conv_scratch_buf_esp32s3(const void *buf); -int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const uint16_t ch_mult, - const uint16_t filter_wd, - const uint16_t filter_ht); +int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const dw_conv_params_t *conv_params); void esp_nn_set_depthwise_conv_scratch_buf_esp32s3(const void *buf); /************************** Pooling functions *****************************/ diff --git a/code/components/esp-nn/include/esp_nn_esp32.h b/code/components/esp-nn/include/esp_nn_generic_opt.h similarity index 77% rename from code/components/esp-nn/include/esp_nn_esp32.h rename to code/components/esp-nn/include/esp_nn_generic_opt.h index 03fd8216..136cba5d 100644 --- a/code/components/esp-nn/include/esp_nn_esp32.h +++ b/code/components/esp-nn/include/esp_nn_generic_opt.h @@ -13,28 +13,27 @@ // limitations under the License. /** - * @file Header definitions to include for esp_nn optimized functions for - * the ESP32 platform. - * We are hooking up just the C versions for now. - * The file hence is exactly same as `esp_nn_ansi_c.h` + * @file Header definitions to include for esp_nn generic optimisations + * For functions which not having optimisations, _ansi versions are picked. */ #pragma once +#include "esp_nn_defs.h" #include "esp_nn_ansi_headers.h" #define esp_nn_add_elementwise_s8 esp_nn_add_elementwise_s8_ansi #define esp_nn_mul_elementwise_s8 esp_nn_mul_elementwise_s8_ansi -#define esp_nn_depthwise_conv_s8 esp_nn_depthwise_conv_s8_ansi +#define esp_nn_depthwise_conv_s8 esp_nn_depthwise_conv_s8_opt -#define esp_nn_conv_s8 esp_nn_conv_s8_ansi +#define esp_nn_conv_s8 esp_nn_conv_s8_opt -#define esp_nn_get_conv_scratch_size esp_nn_get_conv_scratch_size_ansi -#define esp_nn_set_conv_scratch_buf esp_nn_set_conv_scratch_buf_ansi +#define esp_nn_get_conv_scratch_size esp_nn_get_conv_scratch_size_opt +#define esp_nn_set_conv_scratch_buf esp_nn_set_conv_scratch_buf_opt -#define esp_nn_get_depthwise_conv_scratch_size esp_nn_get_depthwise_conv_scratch_size_ansi -#define esp_nn_set_depthwise_conv_scratch_buf esp_nn_set_depthwise_conv_scratch_buf_ansi +#define esp_nn_get_depthwise_conv_scratch_size esp_nn_get_depthwise_conv_scratch_size_opt +#define esp_nn_set_depthwise_conv_scratch_buf esp_nn_set_depthwise_conv_scratch_buf_opt #define esp_nn_relu6_s8 esp_nn_relu6_s8_ansi diff --git a/code/components/esp-nn/src/common/common_functions.h b/code/components/esp-nn/src/common/common_functions.h index 9a5f0dcc..0a74eca4 100644 --- a/code/components/esp-nn/src/common/common_functions.h +++ b/code/components/esp-nn/src/common/common_functions.h @@ -41,15 +41,39 @@ __NN_FORCE_INLINE__ int32_t esp_nn_clz32(uint32_t in) { +#if CONFIG_IDF_TARGET_ARCH_XTENSA __asm__ volatile("nsau %0, %0" : "+r" (in)); return in; -} - -__NN_FORCE_INLINE__ int32_t esp_nn_pick_sat_high32_of64(int64_t val64) -{ - int32_t sign = (int32_t) (val64 >> 63); - int32_t to_add = sign & ((1ul << 31) - 1); - return (int32_t) ((int64_t) (val64 + to_add) >> 31); +#elif defined(__GNUC__) + return __builtin_clz(in); +#else + int32_t count = 32; + uint32_t x = in, y = in >> 16; + if (y != 0) { + count -= 16; + x = y; + } + y = x >> 8; + if (y != 0) { + count -= 8; + x = y; + } + y = x >> 4; + if (y != 0) { + count -= 4; + x = y; + } + y = x >> 2; + if (y != 0) { + count -= 2; + x = y; + } + y = x >> 1; + if (y != 0) { + return count - 2; + } + return count - x; +#endif } /** @@ -57,8 +81,19 @@ __NN_FORCE_INLINE__ int32_t esp_nn_pick_sat_high32_of64(int64_t val64) */ __NN_FORCE_INLINE__ int32_t esp_nn_saturate8(int32_t in) { +#if CONFIG_IDF_TARGET_ARCH_XTENSA __asm__ volatile("clamps %0, %0, 7" : "+a"(in)); return in; +#else + return max(INT8_MIN, min(in, INT8_MAX)); +#endif +} + +__NN_FORCE_INLINE__ int32_t esp_nn_pick_sat_high32_of64(int64_t val64) +{ + int32_t sign = (int32_t) (val64 >> 63); + int32_t to_add = sign & ((1ul << 31) - 1); + return (int32_t) ((int64_t) (val64 + to_add) >> 31); } __NN_FORCE_INLINE__ int32_t esp_nn_sat_round_doubling_high_mul(int32_t in0, int32_t in1) @@ -144,7 +179,7 @@ static void esp_nn_aligned_s8_pad_with_value(const int8_t *src, int8_t *dst, const uint16_t pad_ht) { /* memset with pad_val */ - memset(dst, pad_val, ((input_wd + 2 * pad_wd) * (input_ht + 2 * pad_ht)) * channels * 2); + memset(dst, pad_val, ((input_wd + 2 * pad_wd) * (input_ht + 2 * pad_ht)) * channels); dst += (pad_wd + input_wd + pad_wd) * channels; for (int i = 0; i < input_ht; i++) { @@ -156,7 +191,6 @@ static void esp_nn_aligned_s8_pad_with_value(const int8_t *src, int8_t *dst, } } -#if 0 static void esp_nn_aligned_s8_pad_end_with_value(const int8_t *src, int8_t *dst, const uint16_t input_wd, const uint16_t input_ht, @@ -169,13 +203,16 @@ static void esp_nn_aligned_s8_pad_end_with_value(const int8_t *src, int8_t *dst, for (int j = 0; j < input_wd * channels; j++) { *dst++ = *src++; } - memset(dst, pad_val, pad_wd * channels); - dst += pad_wd * channels; + if (pad_wd) { + memset(dst, pad_val, pad_wd * channels); + dst += pad_wd * channels; + } } /* pad end `pad_ht` lines at end */ - memset(dst, pad_val, (input_wd + pad_wd) * pad_ht * channels); + if (pad_ht) { + memset(dst, pad_val, (input_wd + pad_wd) * pad_ht * channels); + } } -#endif /** * @brief convert 8 bit input data to 16 bit diff --git a/code/components/esp-nn/src/convolution/esp_nn_conv_ansi.c b/code/components/esp-nn/src/convolution/esp_nn_conv_ansi.c index d04f78e1..677c0ad8 100644 --- a/code/components/esp-nn/src/convolution/esp_nn_conv_ansi.c +++ b/code/components/esp-nn/src/convolution/esp_nn_conv_ansi.c @@ -12,16 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include -int esp_nn_get_conv_scratch_size_ansi(const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_ch, - const uint16_t out_ch, - const uint16_t filter_wd, - const uint16_t filter_ht) +int esp_nn_get_conv_scratch_size_ansi(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const conv_params_t *conv_params) { return 0; } @@ -108,29 +106,35 @@ void esp_nn_conv_u8_ansi(const uint8_t *input_data, * Assumption 2: Pointers are valid * Assumption 3: dialation width = 1 */ -void esp_nn_conv_s8_ansi(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, +void esp_nn_conv_s8_ansi(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, + const data_dims_t *output_dims, int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const uint16_t out_channels, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max) + const conv_params_t *conv_params, + const quant_data_t *quant_data) { + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t in_channels = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const uint16_t out_channels = output_dims->channels; + const int32_t *out_shift = quant_data->shift; + const int32_t *out_mult = quant_data->mult; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx; for (out_y = 0; out_y < out_ht; out_y++) { diff --git a/code/components/esp-nn/src/convolution/esp_nn_conv_esp32s3.c b/code/components/esp-nn/src/convolution/esp_nn_conv_esp32s3.c index ea8fdfa5..e13129b2 100644 --- a/code/components/esp-nn/src/convolution/esp_nn_conv_esp32s3.c +++ b/code/components/esp-nn/src/convolution/esp_nn_conv_esp32s3.c @@ -12,30 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include +#include #include static int16_t *scratch_buffer = NULL; -extern void esp_nn_conv_s16_mult8_1x1_esp32s3(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_channels, - const int32_t input_offset, - const int16_t *filter_data, - const int32_t *bias, - int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const uint16_t out_channels, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max, - void *buffer /* scratch buffer */); +extern void esp_nn_conv_s8_mult8_1x1_esp32s3(const int8_t *input_data, + const uint16_t input_wd, + const uint16_t input_ht, + const uint16_t in_channels, + const int32_t input_offset, + const int8_t *filter_aligned, + const int32_t *bias, + int8_t *out_data, + const uint16_t out_wd, + const uint16_t out_ht, + const uint16_t out_channels, + const int32_t out_offset, + const int32_t *out_shift, + const int32_t *out_mult, + const int32_t activation_min, + const int32_t activation_max, + void *buffer /* scratch buffer */); extern void esp_nn_conv_s16_mult4_1x1_esp32s3(const int16_t *input_data, const uint16_t input_wd, @@ -81,34 +81,40 @@ extern void esp_nn_aligned_s8_to_s16_with_offset_esp32s3(const int8_t *src, int1 extern void esp_nn_s8_to_s16_esp32s3(const int8_t *src, int16_t *dst, const int size); -static void esp_nn_conv_s8_unrolled(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, +static void esp_nn_conv_s8_unrolled(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, + const data_dims_t *output_dims, int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const uint16_t out_channels, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max) + const conv_params_t *conv_params, + const quant_data_t *quant_data) { + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t in_ch = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const uint16_t out_ch = output_dims->channels; + const int32_t *out_shift = quant_data->shift; + const int32_t *out_mult = quant_data->mult; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx; for (out_y = 0; out_y < out_ht; out_y++) { for (out_x = 0; out_x < out_wd; out_x++) { - for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) { + for (out_ch_idx = 0; out_ch_idx < out_ch; out_ch_idx++) { int32_t conv_out = 0; const int32_t base_y = stride_ht * out_y - pad_ht; @@ -124,10 +130,10 @@ static void esp_nn_conv_s8_unrolled(const int8_t *input_data, for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) { const int32_t in_row = base_y + filter_y_idx; const int32_t in_col = base_x + filter_x_idx; - int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels; - int32_t filter_base_offset = out_ch_idx * in_channels * filter_ht * filter_wd + - (filter_y_idx * filter_wd + filter_x_idx) * in_channels; - for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) { + int32_t input_base_offset = (in_row * input_wd + in_col) * in_ch; + int32_t filter_base_offset = out_ch_idx * in_ch * filter_ht * filter_wd + + (filter_y_idx * filter_wd + filter_x_idx) * in_ch; + for (in_ch_idx = 0; in_ch_idx < in_ch; in_ch_idx++) { conv_out += (input_data[input_base_offset + in_ch_idx] + input_offset) * filter_data[filter_base_offset + in_ch_idx]; @@ -332,18 +338,35 @@ static void esp_nn_conv_s8_pad_valid_ch3_3x3(const int8_t *input_data, } } -int esp_nn_get_conv_scratch_size_esp32s3(const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t in_ch, - const uint16_t out_ch, - const uint16_t filter_wd, - const uint16_t filter_ht) +int esp_nn_get_conv_scratch_size_esp32s3(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const conv_params_t *conv_params) { + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t in_ch = input_dims->channels; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t out_ch = output_dims->channels; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + int filter_size = filter_wd * filter_ht * in_ch * out_ch; int input_size = input_wd * input_ht * in_ch; - int transpose_buf_size = 8 * in_ch; /* to store intermediate data */ + + int transpose_buf_size = 2 * (8 * in_ch); /* to store intermediate data */ + if (input_wd * input_ht < 8) { + transpose_buf_size = 0; // not using this for leftover + } int align_buf_size = 32; /* extra buffer for alignment */ - return 2 * (filter_size + input_size + transpose_buf_size) + align_buf_size; + if (in_ch % 8 == 0 && filter_wd == 1 && filter_ht == 1 && + pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) { + return filter_size + transpose_buf_size + align_buf_size; + } + return 2 * (filter_size + input_size) + transpose_buf_size + align_buf_size; } void esp_nn_set_conv_scratch_buf_esp32s3(void *buf) @@ -351,29 +374,35 @@ void esp_nn_set_conv_scratch_buf_esp32s3(void *buf) scratch_buffer = (int16_t *) buf; } -void esp_nn_conv_s8_esp32s3(const int8_t *input, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, +void esp_nn_conv_s8_esp32s3(const data_dims_t *input_dims, + const int8_t *input, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, + const data_dims_t *output_dims, int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const uint16_t out_channels, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max) + const conv_params_t *conv_params, + const quant_data_t *quant_data) { + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t channels = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const uint16_t out_channels = output_dims->channels; + const int32_t *out_shift = quant_data->shift; + const int32_t *out_mult = quant_data->mult; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + int filter_size = filter_wd * filter_ht * channels * out_channels; int input_size = input_wd * input_ht * channels; int align_len = 16 - (filter_size & 15); @@ -387,15 +416,16 @@ void esp_nn_conv_s8_esp32s3(const int8_t *input, if (channels % 8 == 0 && filter_wd == 1 && filter_ht == 1 && pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) { - int scratch_offset = (int) (filter_data16 + filter_size); + int8_t *filter_aligned = (int8_t *) scratch_buffer; + int scratch_offset = (int) (filter_aligned + filter_size); void *scratch_buf = (void *) (scratch_offset + 16 - (scratch_offset & 15)); - esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size); - esp_nn_conv_s16_mult8_1x1_esp32s3( - input, input_wd, input_ht, channels, input_offset, filter_data16, + memcpy(filter_aligned, filter_data, filter_size); // copy to aligned address + esp_nn_conv_s8_mult8_1x1_esp32s3( + input, input_wd, input_ht, channels, input_offset, filter_aligned, bias, out_data, out_wd, out_ht, out_channels, out_offset, out_shift, out_mult, activation_min, activation_max, scratch_buf); } else if (channels % 4 == 0 && filter_wd == 1 && filter_ht == 1 && - (input_wd * input_ht) % 16 == 0 && /* TODO: remove this check */ + (input_wd * input_ht) % 4 == 0 && /* TODO: remove this check */ pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) { int scratch_offset = (int) (input_data16 + input_size); void *scratch_buf = (void *) (scratch_offset + 16 - (scratch_offset & 15)); @@ -427,10 +457,7 @@ void esp_nn_conv_s8_esp32s3(const int8_t *input, } } else { /* Basic unrolled version */ - esp_nn_conv_s8_unrolled(input, input_wd, input_ht, channels, input_offset, - pad_wd, pad_ht, stride_wd, stride_ht, - filter_data, filter_wd, filter_ht, bias, - out_data, out_wd, out_ht, out_channels, out_offset, out_shift, - out_mult, activation_min, activation_max); + esp_nn_conv_s8_unrolled(input_dims, input, filter_dims, filter_data, + bias, output_dims, out_data, conv_params, quant_data); } } diff --git a/code/components/esp-nn/src/convolution/esp_nn_conv_opt.c b/code/components/esp-nn/src/convolution/esp_nn_conv_opt.c new file mode 100644 index 00000000..be96430e --- /dev/null +++ b/code/components/esp-nn/src/convolution/esp_nn_conv_opt.c @@ -0,0 +1,179 @@ +// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +int esp_nn_get_conv_scratch_size_opt(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const conv_params_t *conv_params) +{ + return 0; +} + +void esp_nn_set_conv_scratch_buf_opt(const void *buf) +{ + +} + +__attribute__ ((noinline)) +static void esp_nn_conv_s8_1x1(const data_dims_t *input_dims, + const int8_t *input_data, + const int8_t *filter_data, + const int32_t *bias, + const data_dims_t *output_dims, + int8_t *out_data, + const conv_params_t *conv_params, + const quant_data_t *quant_data) +{ + const uint16_t input_wd = input_dims->width; + const uint16_t in_channels = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const uint16_t out_channels = output_dims->channels; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + + for (int32_t in_row = 0; in_row < out_ht * stride_ht; in_row += stride_ht) { + for (int32_t in_col = 0; in_col < out_wd * stride_wd; in_col += stride_wd) { + const int32_t *out_mult = quant_data->mult; + const int32_t *out_shift = quant_data->shift; + const int8_t *filter_ptr = filter_data; + const int8_t *input_base_ptr = input_data + (in_row * input_wd + in_col) * in_channels; + int32_t out_ch_idx = 0; + for (; out_ch_idx < out_channels; out_ch_idx++) { + int32_t conv_out = 0; + + const int8_t *input_ptr = input_base_ptr; + + int32_t in_ch_idx = 0; + for (; in_ch_idx < in_channels - 3; in_ch_idx += 4) { + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + } + for (; in_ch_idx < in_channels; in_ch_idx ++) { + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + } + if (bias) { + conv_out += bias[out_ch_idx]; + } + conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, *out_mult++, *out_shift++); + conv_out += out_offset; + conv_out = max(conv_out, activation_min); + conv_out = min(conv_out, activation_max); + *out_data++ = (int8_t) conv_out; + } + } + } +} + +/** + * Assumption 1: i/p channels == o/p channels + * Assumption 2: Pointers are valid + * Assumption 3: dialation width = 1 + */ +void esp_nn_conv_s8_opt(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, + const int8_t *filter_data, + const int32_t *bias, + const data_dims_t *output_dims, + int8_t *out_data, + const conv_params_t *conv_params, + const quant_data_t *quant_data) +{ + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + + if (filter_wd == 1 && filter_ht == 1) { + esp_nn_conv_s8_1x1(input_dims, input_data, filter_data, bias, + output_dims, out_data, conv_params, quant_data); + return; + } + + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t in_channels = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const uint16_t out_channels = output_dims->channels; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + + int32_t out_ch_idx, out_y, out_x, filter_y_idx, filter_x_idx; + + for (out_y = 0; out_y < out_ht; out_y++) { + for (out_x = 0; out_x < out_wd; out_x++) { + const int32_t *out_shift = quant_data->shift; + const int32_t *out_mult = quant_data->mult; + for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) { + int32_t conv_out = 0; + + const int32_t base_y = stride_ht * out_y - pad_ht; + const int32_t base_x = stride_wd * out_x - pad_wd; + + const int32_t filter_y_start = max(0, -base_y); + const int32_t filter_x_start = max(0, -base_x); + + const int32_t filter_y_end = min(filter_ht, input_ht - base_y); + const int32_t filter_x_end = min(filter_wd, input_wd - base_x); + + for (filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) { + for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) { + const int32_t in_row = base_y + filter_y_idx; + const int32_t in_col = base_x + filter_x_idx; + + const int8_t *input_ptr = input_data + + (in_row * input_wd + in_col) * in_channels; + const int8_t *filter_ptr = filter_data + + out_ch_idx * in_channels * filter_ht * filter_wd + + (filter_y_idx * filter_wd + filter_x_idx) * in_channels; + int32_t in_ch_idx = 0; + for (; in_ch_idx < in_channels - 3; in_ch_idx += 4) { + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + } + for (; in_ch_idx < in_channels; in_ch_idx ++) { + conv_out += (*input_ptr++ + input_offset) * *filter_ptr++; + } + } + } + if (bias) { + conv_out += bias[out_ch_idx]; + } + conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, *out_mult++, *out_shift++); + conv_out += out_offset; + conv_out = max(conv_out, activation_min); + conv_out = min(conv_out, activation_max); + *out_data++ = (int8_t) conv_out; + } + } + } +} diff --git a/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_ansi.c b/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_ansi.c index 9cac6cef..1cd02e0f 100644 --- a/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_ansi.c +++ b/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_ansi.c @@ -12,16 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - +#include #include -int esp_nn_get_depthwise_conv_scratch_size_ansi(const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const uint16_t ch_mult, - const uint16_t filter_wd, - const uint16_t filter_ht) +int esp_nn_get_depthwise_conv_scratch_size_ansi(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const dw_conv_params_t *conv_params) { return 0; } @@ -31,29 +28,35 @@ void esp_nn_set_depthwise_conv_scratch_buf_ansi(const void *buf) } -void esp_nn_depthwise_conv_s8_ansi(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, - const uint16_t ch_mult, +void esp_nn_depthwise_conv_s8_ansi(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, + const data_dims_t *output_dims, int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max) + const dw_conv_params_t *conv_params, + const quant_data_t *quant_data) { + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t channels = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const int32_t *out_shift = quant_data->shift; + const int32_t *out_mult = quant_data->mult; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + const uint16_t ch_mult = conv_params->ch_mult; + int out_idx = 0; for (int out_y = 0; out_y < out_ht; out_y++) { //height loop const int16_t base_y = (out_y * stride_ht) - pad_ht; diff --git a/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_opt.c b/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_opt.c new file mode 100644 index 00000000..4afea3f3 --- /dev/null +++ b/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_opt.c @@ -0,0 +1,291 @@ +// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +int esp_nn_get_depthwise_conv_scratch_size_opt(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const dw_conv_params_t *conv_params) +{ + return 0; +} + +void esp_nn_set_depthwise_conv_scratch_buf_opt(const void *buf) +{ + +} + +/* common channel multiplier == 1 case */ +__attribute__ ((noinline)) +static void esp_nn_depthwise_conv_s8_ch_mult_1(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, + const int8_t *filter_data, + const int32_t *bias, + const data_dims_t *output_dims, + int8_t *out_data, + const dw_conv_params_t *conv_params, + const quant_data_t *quant_data) +{ + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t channels = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + + int out_idx = 0; + for (int out_y = 0; out_y < out_ht; out_y++) { //height loop + const int16_t base_y = (out_y * stride_ht) - pad_ht; + for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop + const int16_t base_x = (out_x * stride_wd) - pad_wd; + + const int32_t *out_shift = quant_data->shift; + const int32_t *out_mult = quant_data->mult; + + /* Select filter so as the point doesn't lie outside block */ + int filter_y_start = max(0, -base_y); + int filter_x_start = max(0, -base_x); + int filter_y_end = min(filter_ht, input_ht - base_y); + int filter_x_end = min(filter_wd, input_wd - base_x); + + int ch_idx = 0; + for (; ch_idx < channels - 3; ch_idx += 4) {//channel_loop + int32_t result0 = 0; + int32_t result1 = 0; + int32_t result2 = 0; + int32_t result3 = 0; + + for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) { + const int32_t idx_y = base_y + filter_y_idx; + for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) { + const int32_t idx_x = base_x + filter_x_idx; + int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx; + int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels) + ch_idx; + int32_t input_val0 = input_data[input_index + 0] + input_offset; + int32_t input_val1 = input_data[input_index + 1] + input_offset; + int32_t input_val2 = input_data[input_index + 2] + input_offset; + int32_t input_val3 = input_data[input_index + 3] + input_offset; + int32_t filter_val0 = filter_data[filter_index + 0]; + int32_t filter_val1 = filter_data[filter_index + 1]; + int32_t filter_val2 = filter_data[filter_index + 2]; + int32_t filter_val3 = filter_data[filter_index + 3]; + result0 += input_val0 * filter_val0; + result1 += input_val1 * filter_val1; + result2 += input_val2 * filter_val2; + result3 += input_val3 * filter_val3; + } + } + if (bias) { + result0 += bias[ch_idx + 0]; + result1 += bias[ch_idx + 1]; + result2 += bias[ch_idx + 2]; + result3 += bias[ch_idx + 3]; + } + result0 = esp_nn_multiply_by_quantized_mult_fast(result0, *out_mult++, *out_shift++); + result1 = esp_nn_multiply_by_quantized_mult_fast(result1, *out_mult++, *out_shift++); + result2 = esp_nn_multiply_by_quantized_mult_fast(result2, *out_mult++, *out_shift++); + result3 = esp_nn_multiply_by_quantized_mult_fast(result3, *out_mult++, *out_shift++); + + result0 += out_offset; + result1 += out_offset; + result2 += out_offset; + result3 += out_offset; + + result0 = max(result0, activation_min); + result1 = max(result1, activation_min); + result2 = max(result2, activation_min); + result3 = max(result3, activation_min); + + result0 = min(result0, activation_max); + result1 = min(result1, activation_max); + result2 = min(result2, activation_max); + result3 = min(result3, activation_max); + + out_data[out_idx++] = result0; + out_data[out_idx++] = result1; + out_data[out_idx++] = result2; + out_data[out_idx++] = result3; + } + for (; ch_idx < channels; ch_idx++) {//channel_loop + int32_t result = 0; + + for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) { + const int32_t idx_y = base_y + filter_y_idx; + for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) { + const int32_t idx_x = base_x + filter_x_idx; + int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx; + int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels) + ch_idx; + int32_t input_val = input_data[input_index] + input_offset; + int32_t filter_val = filter_data[filter_index]; + result += input_val * filter_val; + } + } + if (bias) { + result += bias[ch_idx]; + } + result = esp_nn_multiply_by_quantized_mult_fast(result, *out_mult++, *out_shift++); + result += out_offset; + result = max(result, activation_min); + result = min(result, activation_max); + + out_data[out_idx++] = result; + } + } + } +} + +void esp_nn_depthwise_conv_s8_opt(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, + const int8_t *filter_data, + const int32_t *bias, + const data_dims_t *output_dims, + int8_t *out_data, + const dw_conv_params_t *conv_params, + const quant_data_t *quant_data) +{ + const uint16_t ch_mult = conv_params->ch_mult; + if (ch_mult == 1) { + esp_nn_depthwise_conv_s8_ch_mult_1(input_dims, input_data, filter_dims, filter_data, + bias, output_dims, out_data, conv_params, quant_data); + return; + } + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t channels = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + + int out_idx = 0; + for (int out_y = 0; out_y < out_ht; out_y++) { //height loop + const int16_t base_y = (out_y * stride_ht) - pad_ht; + for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop + const int16_t base_x = (out_x * stride_wd) - pad_wd; + + const int32_t *out_shift = quant_data->shift; + const int32_t *out_mult = quant_data->mult; + + /* Select filter so as the point doesn't lie outside block */ + int filter_y_start = max(0, -base_y); + int filter_x_start = max(0, -base_x); + int filter_y_end = min(filter_ht, input_ht - base_y); + int filter_x_end = min(filter_wd, input_wd - base_x); + + for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop + int ch_mult_idx = 0; + for (; ch_mult_idx < ch_mult - 3; ch_mult_idx += 4) { + int32_t result0 = 0; + int32_t result1 = 0; + int32_t result2 = 0; + int32_t result3 = 0; + const int out_ch_idx = ch_idx * ch_mult + ch_mult_idx; + + for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) { + const int32_t idx_y = base_y + filter_y_idx; + for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) { + const int32_t idx_x = base_x + filter_x_idx; + int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx; + int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx; + int32_t input_val = input_data[input_index] + input_offset; + int32_t filter_val0 = filter_data[filter_index + 0]; + int32_t filter_val1 = filter_data[filter_index + 1]; + int32_t filter_val2 = filter_data[filter_index + 2]; + int32_t filter_val3 = filter_data[filter_index + 3]; + result0 += input_val * filter_val0; + result1 += input_val * filter_val1; + result2 += input_val * filter_val2; + result3 += input_val * filter_val3; + } + } + if (bias) { + result0 += bias[out_ch_idx + 0]; + result1 += bias[out_ch_idx + 1]; + result2 += bias[out_ch_idx + 2]; + result3 += bias[out_ch_idx + 3]; + } + result0 = esp_nn_multiply_by_quantized_mult_fast(result0, *out_mult++, *out_shift++); + result1 = esp_nn_multiply_by_quantized_mult_fast(result1, *out_mult++, *out_shift++); + result2 = esp_nn_multiply_by_quantized_mult_fast(result2, *out_mult++, *out_shift++); + result3 = esp_nn_multiply_by_quantized_mult_fast(result3, *out_mult++, *out_shift++); + + result0 += out_offset; + result1 += out_offset; + result2 += out_offset; + result3 += out_offset; + + result0 = max(result0, activation_min); + result1 = max(result1, activation_min); + result2 = max(result2, activation_min); + result3 = max(result3, activation_min); + result0 = min(result0, activation_max); + result1 = min(result1, activation_max); + result2 = min(result2, activation_max); + result3 = min(result3, activation_max); + + out_data[out_idx++] = result0; + out_data[out_idx++] = result1; + out_data[out_idx++] = result2; + out_data[out_idx++] = result3; + } + for (; ch_mult_idx < ch_mult; ch_mult_idx++) { + int32_t result = 0; + const int out_ch_idx = ch_idx * ch_mult + ch_mult_idx; + + for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) { + const int32_t idx_y = base_y + filter_y_idx; + for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) { + const int32_t idx_x = base_x + filter_x_idx; + int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx; + int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx; + int32_t input_val = input_data[input_index] + input_offset; + int32_t filter_val = filter_data[filter_index]; + result += input_val * filter_val; + } + } + if (bias) { + result += bias[out_ch_idx]; + } + result = esp_nn_multiply_by_quantized_mult_fast(result, *out_mult++, *out_shift++); + result += out_offset; + result = max(result, activation_min); + result = min(result, activation_max); + + out_data[out_idx++] = result; + } + } + } + } +} diff --git a/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_s8_esp32s3.c b/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_s8_esp32s3.c index c588c48f..9167a43f 100644 --- a/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_s8_esp32s3.c +++ b/code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_s8_esp32s3.c @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include +#include #include @@ -353,17 +353,59 @@ void esp_nn_depthwise_conv_s8_ch_mult1(const int8_t *input_data, } } -int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const uint16_t ch_mult, - const uint16_t filter_wd, - const uint16_t filter_ht) +int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const data_dims_t *input_dims, + const data_dims_t *filter_dims, + const data_dims_t *output_dims, + const dw_conv_params_t *conv_params) { + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t channels = input_dims->channels; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t ch_mult = conv_params->ch_mult; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + int filter_size = filter_wd * filter_ht * channels * ch_mult; - int padding_used = ((filter_wd == 3) && (filter_ht == 3)) * 2; - int input_size = (input_wd + padding_used) * (input_ht + padding_used) * channels; - return 2 * (filter_size + input_size) + 16; //16 for alignment + int pad_width = 0, pad_height = 0; + + if ((ch_mult == 1) && (channels % 8 == 0) && (filter_wd == 3) && (filter_ht == 3)) { + if (channels % 16 == 0) { + if (pad_wd || pad_ht) { + pad_width = pad_wd * 2; + pad_height = pad_ht * 2; + } else { + // check if we need to pad additionally + pad_width = (out_wd * stride_wd + filter_wd - 1) - input_wd; + pad_height = (out_ht * stride_ht + filter_ht - 1) - input_ht; + // printf("in(%d %d %d), out(%d %d), filter (%d %d) stride (%d %d), pad (%d %d)", + // input_wd, input_ht, channels, out_wd, out_ht, filter_wd, filter_ht, + // stride_wd, stride_ht, pad_wd, pad_ht); + } + if (pad_width || pad_height) { + int input_size = (input_wd + pad_width) * (input_ht + pad_height) * channels; + // printf("ask1 %d\n", filter_size + input_size + 16); + return filter_size + input_size + 16; // 16 for alignment + } else { + // printf("ask2 %d\n", filter_size + 16); + return filter_size + 16; // 16 for alignment + } + } else { + int input_size = input_wd * input_ht * channels; + // printf("ask3 %d\n", 2 * (filter_size + input_size) + 16); + return 2 * (filter_size + input_size) + 16; // 16 for alignment + } + } else if (ch_mult % 4 == 0) { + int input_size = input_wd * input_ht * channels; + // printf("ask4 %d\n", 2 * (filter_size + input_size) + 16); + return 2 * (filter_size + input_size) + 16; // 16 for alignment + } + return 32; // just few bytes } void esp_nn_set_depthwise_conv_scratch_buf_esp32s3(void *buf) @@ -376,29 +418,38 @@ void esp_nn_set_depthwise_conv_scratch_buf_esp32s3(void *buf) * Assumption 2: Pointers are valid * Assumption 3: dialation width = 1 */ -void esp_nn_depthwise_conv_s8_esp32s3(const int8_t *input_data, - const uint16_t input_wd, - const uint16_t input_ht, - const uint16_t channels, - const int32_t input_offset, - const uint16_t pad_wd, - const uint16_t pad_ht, - const uint16_t stride_wd, - const uint16_t stride_ht, - const uint16_t ch_mult, + + + +void esp_nn_depthwise_conv_s8_esp32s3(const data_dims_t *input_dims, + const int8_t *input_data, + const data_dims_t *filter_dims, const int8_t *filter_data, - const uint16_t filter_wd, - const uint16_t filter_ht, const int32_t *bias, + const data_dims_t *output_dims, int8_t *out_data, - const uint16_t out_wd, - const uint16_t out_ht, - const int32_t out_offset, - const int32_t *out_shift, - const int32_t *out_mult, - const int32_t activation_min, - const int32_t activation_max) + const dw_conv_params_t *conv_params, + const quant_data_t *quant_data) { + const uint16_t input_wd = input_dims->width; + const uint16_t input_ht = input_dims->height; + const uint16_t channels = input_dims->channels; + const int32_t input_offset = conv_params->in_offset; + const int32_t out_offset = conv_params->out_offset; + const uint16_t pad_wd = conv_params->padding.width; + const uint16_t pad_ht = conv_params->padding.height; + const uint16_t stride_wd = conv_params->stride.width; + const uint16_t stride_ht = conv_params->stride.height; + const uint16_t filter_wd = filter_dims->width; + const uint16_t filter_ht = filter_dims->height; + const uint16_t out_wd = output_dims->width; + const uint16_t out_ht = output_dims->height; + const int32_t *out_shift = quant_data->shift; + const int32_t *out_mult = quant_data->mult; + const int32_t activation_min = conv_params->activation.min; + const int32_t activation_max = conv_params->activation.max; + const uint16_t ch_mult = conv_params->ch_mult; + int filter_size = filter_wd * filter_ht * channels * ch_mult; int align_len = 16 - (filter_size & 15); int input_size = input_wd * input_ht * channels; @@ -423,18 +474,27 @@ void esp_nn_depthwise_conv_s8_esp32s3(const int8_t *input_data, stride_wd, stride_ht, filter_aligned, bias, out_data, out_wd, out_ht, out_offset, out_shift, out_mult, activation_min, activation_max); - } else if ((pad_wd == 0) && (pad_ht == 0) && - // because this does not handle padding offset cases yet, run just for stride (1, 1). - // end padding of input with `-input_offset` should solve this - (stride_wd == 1) && (stride_ht == 1)) { + } else if ((channels % 16 == 0) && (pad_wd == 0) && (pad_ht == 0)) { /* process in 8 bits */ int8_t *filter_aligned = (int8_t *) scratch_buffer; + int8_t *input_padded = (int8_t *) scratch_buffer + filter_size + align_len; + + // check if we need to pad additionally + int pad_right = (out_wd * stride_wd + filter_wd - 1) - input_wd; + int pad_bottom = (out_ht * stride_ht + filter_ht - 1) - input_ht; + if (pad_right || pad_bottom) { // pad right and bottom + esp_nn_aligned_s8_pad_end_with_value(input_data, input_padded, input_wd, input_ht, + channels, -input_offset, pad_right, pad_bottom); + } else { + input_padded = (int8_t *) input_data; + } memcpy(filter_aligned, filter_data, filter_size); - esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(input_data, input_wd, input_ht, channels, input_offset, - stride_wd, stride_ht, filter_aligned, - bias, out_data, out_wd, out_ht, out_offset, out_shift, + esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(input_padded, input_wd + pad_right, + input_ht + pad_bottom, channels, input_offset, + stride_wd, stride_ht, filter_aligned, bias, + out_data, out_wd, out_ht, out_offset, out_shift, out_mult, activation_min, activation_max); - } else { /* (channels % 8) == 0 && pad_wd == 1 && pad_ht == 1 */ + } else { /* (channels % 8) == 0 */ esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size); esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input_data, input_data16, input_size, input_offset); esp_nn_depthwise_conv_s16_mult1_3x3_esp32s3(input_data16, input_wd, input_ht, channels, diff --git a/code/components/esp-nn/test_app/sdkconfig.defaults.esp32s3 b/code/components/esp-nn/test_app/sdkconfig.defaults.esp32s3 new file mode 100644 index 00000000..1adc4b01 --- /dev/null +++ b/code/components/esp-nn/test_app/sdkconfig.defaults.esp32s3 @@ -0,0 +1,8 @@ +# Default configurations for ESP32-S3 + +CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32S3_SPIRAM_SUPPORT=y + +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_8WAYS=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y diff --git a/code/components/esp-nn/tests/src/basic_math_test.c b/code/components/esp-nn/tests/src/basic_math_test.c index 5b96b990..715d7c78 100644 --- a/code/components/esp-nn/tests/src/basic_math_test.c +++ b/code/components/esp-nn/tests/src/basic_math_test.c @@ -23,7 +23,9 @@ #include "test_utils.h" #if CONFIG_IDF_CMAKE +#if (CONFIG_SPIRAM_SUPPORT && (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC)) #define IDF_HEAP_CAPS 1 +#endif #if IDF_HEAP_CAPS #include "esp_heap_caps.h" @@ -138,6 +140,11 @@ void esp_nn_add_elementwise_s8_test() out_c_orig = out_data_c; out_opt_orig = out_data_opt; #endif + if (input1_orig == NULL || input2_orig == NULL || out_c_orig == NULL || + out_opt_orig == NULL) { + printf(ANSI_COLOR_RED"%s error allocating buffers\n"ANSI_COLOR_RESET, __FUNCTION__); + goto elementwise_add_test_cleanup; + } for (int i = 0; i < size; ++i) { input1[i] = rand() % 256 - 128; @@ -194,10 +201,10 @@ elementwise_add_test_cleanup: if (input2_orig) { free(input2_orig); } - if (out_data_c) { + if (out_c_orig) { free(out_c_orig); } - if (out_data_opt) { + if (out_opt_orig) { free(out_opt_orig); } } @@ -282,6 +289,11 @@ void esp_nn_mul_elementwise_s8_test() out_c_orig = out_data_c; out_opt_orig = out_data_opt; #endif + if (input1_orig == NULL || input2_orig == NULL || out_c_orig == NULL || + out_opt_orig == NULL) { + printf(ANSI_COLOR_RED"%s error allocating buffers\n"ANSI_COLOR_RESET, __FUNCTION__); + goto elementwise_mult_test_cleanup; + } for (int i = 0; i < size; ++i) { input1[i] = rand() % 256 - 128; @@ -333,10 +345,10 @@ elementwise_mult_test_cleanup: if (input2_orig) { free(input2_orig); } - if (out_data_c) { + if (out_c_orig) { free(out_c_orig); } - if (out_data_opt) { + if (out_opt_orig) { free(out_opt_orig); } } diff --git a/code/components/esp-nn/tests/src/convolution_test.c b/code/components/esp-nn/tests/src/convolution_test.c index f3802257..c86bdbab 100644 --- a/code/components/esp-nn/tests/src/convolution_test.c +++ b/code/components/esp-nn/tests/src/convolution_test.c @@ -22,8 +22,9 @@ #include "test_utils.h" #if CONFIG_IDF_CMAKE +#if (CONFIG_SPIRAM_SUPPORT && (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC)) #define IDF_HEAP_CAPS 1 - +#endif #if IDF_HEAP_CAPS #include "esp_heap_caps.h" #endif @@ -44,8 +45,8 @@ void esp_nn_depthwise_conv_s8_test() uint16_t filter_ht, filter_wd, ch_mult; uint16_t pad_wd, pad_ht, stride_wd, stride_ht; - // run for 10 iterations - for (int itr = 0; itr < 10; itr++) { + // run for 15 iterations + for (int itr = 0; itr < 15; itr++) { /* prepare data */ switch (itr) { case 0: // (ch_mult 1, (channels % 16) = 0), filter (3,3), pad (0,0) @@ -144,22 +145,52 @@ void esp_nn_depthwise_conv_s8_test() stride_wd = 2; stride_ht = 2; break; - default: - input_wd = 4; - input_ht = 4; + case 8: // same as case 7, with large parameters + input_wd = 58; + input_ht = 58; filter_ht = 3; filter_wd = 3; - ch_mult = 4; - channels = 4; - pad_wd = 1; - pad_ht = 1; - stride_wd = 1; - stride_ht = 1; + ch_mult = 1; + channels = 128; + pad_wd = 0; + pad_ht = 0; + stride_wd = 2; + stride_ht = 2; + break; + case 9: // (ch_mult 1, (channels % 16) = 0), filter (3,3), pad (0,0) stride (2,2) + input_wd = 6; + input_ht = 6; + filter_ht = 3; + filter_wd = 3; + ch_mult = 1; + channels = 16; + pad_wd = 0; + pad_ht = 0; + stride_wd = 2; + stride_ht = 2; + break; + default: + input_wd = 6; + input_ht = 6; + filter_ht = 3; + filter_wd = 3; + ch_mult = 1; + channels = 16; + stride_wd = rand() % 2 + 1; + stride_ht = stride_wd; + pad_wd = stride_wd == 1 ? 0 : rand() % 2; + pad_ht = pad_wd; + printf("stride(%d), pad (%d)\t", stride_wd, pad_wd); break; } uint16_t out_wd = (input_wd - filter_wd + 1) / stride_wd; uint16_t out_ht = (input_ht - filter_ht + 1) / stride_ht; + if (itr == 9) { + // expect the function to handle this gracefully + out_wd += 1; + out_ht += 1; + } int in_size = input_wd * input_ht * channels; int out_size = out_wd * out_ht * channels * ch_mult; int filter_size = filter_wd * filter_ht * channels * ch_mult + 4; @@ -210,9 +241,16 @@ void esp_nn_depthwise_conv_s8_test() out_mult[i] = 0x7eb0e200 + rand() % 50; } - int scratch_buf_size = esp_nn_get_depthwise_conv_scratch_size(input_wd, input_ht, - channels, ch_mult, - filter_wd, filter_ht); + data_dims_t input_dims = {.width = input_wd, .height = input_ht, .channels = channels, 1}; + data_dims_t output_dims = {.width = out_wd, .height = out_ht, .channels = channels * ch_mult, 1}; + data_dims_t filter_dims = {.width = filter_wd, .height = filter_ht, 0, 0}; + dw_conv_params_t conv_params = {.in_offset = input_offset, .out_offset = out_offset, .ch_mult = ch_mult, + .stride = {stride_wd, stride_ht}, .padding = {pad_wd, pad_ht}, + .dilation = {0, 0}, .activation = {activation_min, activation_max}}; + quant_data_t quant_data = {.shift = out_shift, .mult = out_mult}; + + int scratch_buf_size = esp_nn_get_depthwise_conv_scratch_size(&input_dims, &filter_dims, + &output_dims, &conv_params); if (scratch_buf_size > 0) { #if IDF_HEAP_CAPS scratch_buf = heap_caps_malloc(scratch_buf_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); @@ -234,11 +272,8 @@ void esp_nn_depthwise_conv_s8_test() } /* C function */ - esp_nn_depthwise_conv_s8_ansi(input, input_wd, input_ht, channels, input_offset, - pad_wd, pad_ht, stride_wd, stride_ht, ch_mult, - filter_data + 4, filter_wd, filter_ht, - bias + 1, out_data_c, out_wd, out_ht, out_offset, out_shift, - out_mult, activation_min, activation_max); + esp_nn_depthwise_conv_s8_ansi(&input_dims, input, &filter_dims, filter_data + 4, + bias + 1, &output_dims, out_data_c, &conv_params, &quant_data); if (itr == 0) { profile_c_end(); @@ -246,11 +281,8 @@ void esp_nn_depthwise_conv_s8_test() } /* Optimized function */ - esp_nn_depthwise_conv_s8(input, input_wd, input_ht, channels, input_offset, - pad_wd, pad_ht, stride_wd, stride_ht, ch_mult, - filter_data + 4, filter_wd, filter_ht, - bias + 1, out_data_opt, out_wd, out_ht, out_offset, out_shift, - out_mult, activation_min, activation_max); + esp_nn_depthwise_conv_s8(&input_dims, input, &filter_dims, filter_data + 4, + bias + 1, &output_dims, out_data_opt, &conv_params, &quant_data); if (itr == 0) { /* disable profiler */ @@ -479,8 +511,16 @@ void esp_nn_conv_s8_test() out_mult[i] = 0x7f67f4f8 + rand() % 50; } - int scratch_buf_size = esp_nn_get_conv_scratch_size(in_wd, in_ht, in_channels, - out_channels, filter_wd, filter_ht); + data_dims_t input_dims = {.width = in_wd, .height = in_ht, .channels = in_channels, 1}; + data_dims_t output_dims = {.width = out_wd, .height = out_ht, .channels = out_channels, 1}; + data_dims_t filter_dims = {.width = filter_wd, .height = filter_ht, 0, 0}; + conv_params_t conv_params = {.in_offset = input_offset, .out_offset = out_offset, + .stride = {stride_wd, stride_ht}, .padding = {pad_wd, pad_ht}, + .dilation = {0, 0}, .activation = {activation_min, activation_max}}; + quant_data_t quant_data = {.shift = out_shift, .mult = out_mult}; + + int scratch_buf_size = esp_nn_get_conv_scratch_size(&input_dims, &filter_dims, + &output_dims, &conv_params); if (scratch_buf_size > 0) { #if IDF_HEAP_CAPS void *scratch_buf = heap_caps_malloc(scratch_buf_size + 32, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); @@ -502,11 +542,8 @@ void esp_nn_conv_s8_test() } /* C function */ - esp_nn_conv_s8_ansi(input, in_wd, in_ht, in_channels, input_offset, - pad_wd, pad_ht, stride_wd, stride_ht, - filter_data + 2, filter_wd, filter_ht, bias, - out_data_c, out_wd, out_ht, out_channels, out_offset, out_shift, - out_mult, activation_min, activation_max); + esp_nn_conv_s8_ansi(&input_dims, input, &filter_dims, filter_data + 2, + bias, &output_dims, out_data_c, &conv_params, &quant_data); if (itr == 0) { profile_c_end(); @@ -514,11 +551,8 @@ void esp_nn_conv_s8_test() } /* Optimized function */ - esp_nn_conv_s8(input, in_wd, in_ht, in_channels, input_offset, - pad_wd, pad_ht, stride_wd, stride_ht, - filter_data + 2, filter_wd, filter_ht, bias, - out_data_opt, out_wd, out_ht, out_channels, out_offset, out_shift, - out_mult, activation_min, activation_max); + esp_nn_conv_s8(&input_dims, input, &filter_dims, filter_data + 2, + bias, &output_dims, out_data_opt, &conv_params, &quant_data); if (itr == 0) { /* disable profiler */ diff --git a/code/components/esp-nn_20220724.zip b/code/components/esp-nn_20220724.zip deleted file mode 100644 index 2bac7498..00000000 Binary files a/code/components/esp-nn_20220724.zip and /dev/null differ diff --git a/code/components/esp-nn_20220716.zip b/code/components/esp-nn_20220827.zip similarity index 97% rename from code/components/esp-nn_20220716.zip rename to code/components/esp-nn_20220827.zip index 53c7bef2..43f16002 100644 Binary files a/code/components/esp-nn_20220716.zip and b/code/components/esp-nn_20220827.zip differ diff --git a/code/components/esp32-camera-master.zip b/code/components/esp32-camera-master.zip deleted file mode 100644 index 8706b3d8..00000000 Binary files a/code/components/esp32-camera-master.zip and /dev/null differ diff --git a/code/components/esp32-camera-master_20220724.zip b/code/components/esp32-camera-master_20220724.zip deleted file mode 100644 index 64f2f896..00000000 Binary files a/code/components/esp32-camera-master_20220724.zip and /dev/null differ diff --git a/code/components/jomjol_controlcamera/ClassControllCamera.cpp b/code/components/jomjol_controlcamera/ClassControllCamera.cpp index df42a2d7..ff410da3 100644 --- a/code/components/jomjol_controlcamera/ClassControllCamera.cpp +++ b/code/components/jomjol_controlcamera/ClassControllCamera.cpp @@ -263,6 +263,9 @@ void CCamera::EnableAutoExposure(int flashdauer) ESP_LOGE(TAGCAMERACLASS, "Camera Capture Failed"); LEDOnOff(false); LightOnOff(false); + LogFile.SwitchOnOff(true); + LogFile.WriteToFile("Camera Capture Failed (Procedure 'EnableAutoExposure') --> Reboot" + "Check that your camera module is working and connected properly."); doReboot(); } esp_camera_fb_return(fb); @@ -313,7 +316,7 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay) LightOnOff(false); LogFile.SwitchOnOff(true); - LogFile.WriteToFile("Camera is not working anymore - most propably hardware problem (instablility, ...). " + LogFile.WriteToFile("Camera is not working anymore (CCamera::CaptureToBasisImage) - most propably hardware problem (instablility, ...). " "System will reboot."); doReboot(); @@ -410,6 +413,9 @@ esp_err_t CCamera::CaptureToFile(std::string nm, int delay) ESP_LOGE(TAGCAMERACLASS, "CaptureToFile: Camera Capture Failed"); LEDOnOff(false); LightOnOff(false); + LogFile.SwitchOnOff(true); + LogFile.WriteToFile("Camera Capture Failed (CCamera::CaptureToFile) --> Reboot" + "Check that your camera module is working and connected properly."); doReboot(); return ESP_FAIL; diff --git a/code/components/jomjol_fileserver_ota/server_ota.cpp b/code/components/jomjol_fileserver_ota/server_ota.cpp index 88d301ae..c9b11b08 100644 --- a/code/components/jomjol_fileserver_ota/server_ota.cpp +++ b/code/components/jomjol_fileserver_ota/server_ota.cpp @@ -416,6 +416,8 @@ void task_reboot(void *pvParameter) } void doReboot(){ + LogFile.SwitchOnOff(true); + LogFile.WriteToFile("Reboot triggert by Software (5s)."); ESP_LOGI(TAGPARTOTA, "Reboot in 5sec"); LogFile.WriteToFile("Reboot in 5sec"); xTaskCreate(&task_reboot, "reboot", configMINIMAL_STACK_SIZE * 64, NULL, 10, NULL); @@ -435,7 +437,7 @@ esp_err_t handler_reboot(httpd_req_t *req) LogFile.WriteToFile("handler_reboot"); ESP_LOGI(TAGPARTOTA, "!!! System will restart within 5 sec!!!"); - const char* resp_str = "!!! System will restart within 5 sec!!!"; + const char* resp_str = "

"; httpd_resp_send(req, resp_str, strlen(resp_str)); doReboot(); diff --git a/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp b/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp index 83243e30..da6ddfd5 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.cpp @@ -42,7 +42,7 @@ string ClassFlowCNNGeneral::getReadout(int _analog = 0, bool _extendedResolution int ergebnis_nachkomma = ((int) floor(zahl * 10) + 10) % 10; prev = ZeigerEvalAnalogNeu(GENERAL[_analog]->ROI[GENERAL[_analog]->ROI.size() - 1]->result_float, prev); - if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::getReadout(analog) zahl=" + std::to_string(zahl) + ", ergebnis_nachkomma=" + std::to_string(ergebnis_nachkomma) + ", prev=" + std::to_string(prev)); +// if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::getReadout(analog) zahl=" + std::to_string(zahl) + ", ergebnis_nachkomma=" + std::to_string(ergebnis_nachkomma) + ", prev=" + std::to_string(prev)); result = std::to_string(prev); if (_extendedResolution && (CNNType != Digital)) @@ -82,8 +82,6 @@ string ClassFlowCNNGeneral::getReadout(int _analog = 0, bool _extendedResolution result = std::to_string(ergebnis_vorkomma) + std::to_string(ergebnis_nachkomma); prev = ergebnis_vorkomma; if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::getReadout(dig100-ext) ergebnis_vorkomma=" + std::to_string(ergebnis_vorkomma) + ", ergebnis_nachkomma=" + std::to_string(ergebnis_nachkomma) + ", prev=" + std::to_string(prev)); - - } else { @@ -129,6 +127,7 @@ string ClassFlowCNNGeneral::getReadout(int _analog = 0, bool _extendedResolution return result; } +/* int ClassFlowCNNGeneral::ZeigerEvalHybrid(float zahl, float zahl_vorgaenger, int eval_vorgaenger) { if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalHybrid( " + std::to_string(zahl) + ", " + std::to_string(zahl_vorgaenger) + ", " + std::to_string(eval_vorgaenger) + ")"); @@ -179,6 +178,7 @@ int ClassFlowCNNGeneral::ZeigerEvalHybrid(float zahl, float zahl_vorgaenger, int return -1; } +*/ int ClassFlowCNNGeneral::ZeigerEvalHybridNeu(float zahl, float zahl_vorgaenger, int eval_vorgaenger, bool AnalogerVorgaenger) { @@ -200,28 +200,17 @@ int ClassFlowCNNGeneral::ZeigerEvalHybridNeu(float zahl, float zahl_vorgaenger, if (AnalogerVorgaenger) { - if (zahl_vorgaenger <= DigitalAnalogerVorgaengerUebergangsbereich) // Nulldurchgang hat stattgefunden - { - result = (int) ((int) round(zahl) + 10) % 10; - if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalHybridNeu - Analoger Vorgänger, Nulldurchgang stattgefunden = " + std::to_string(result) + - " zahl: " + std::to_string(zahl) + " zahl_vorgaenger = " + std::to_string(zahl_vorgaenger)+ " eval_vorgaenger = " + std::to_string(eval_vorgaenger) + " DigitalUnschaerfe = " + std::to_string(DigitalUnschaerfe)); - return result; - } - - if ((ergebnis_nachkomma <= 2) || (ergebnis_nachkomma >= 8)) // Band um die Ziffer --> Runden, da Ziffer im Rahmen Ungenauigkeit erreicht - result = ((int) round(zahl) + 10) % 10; - else - result = ((int) trunc(zahl) + 10) % 10; - - if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalHybridNeu - Analoger Vorgänger, Nulldurchgang NICHT stattgefunden = " + std::to_string(result) + +// result = ZeigerEvalAnalogToDigitNeu(zahl, eval_vorgaenger); + result = ZeigerEvalAnalogToDigitNeu(zahl, zahl_vorgaenger); + if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalHybridNeu - Analoger Vorgänger, Bewertung über ZeigerEvalAnalogNeu = " + std::to_string(result) + " zahl: " + std::to_string(zahl) + " zahl_vorgaenger = " + std::to_string(zahl_vorgaenger)+ " eval_vorgaenger = " + std::to_string(eval_vorgaenger) + " DigitalUnschaerfe = " + std::to_string(DigitalUnschaerfe)); return result; } if ((zahl_vorgaenger >= DigitalUebergangsbereichVorgaenger ) && (zahl_vorgaenger <= (10.0 - DigitalUebergangsbereichVorgaenger))) { - // kein Ziffernwechsel, da Vorkomma weit genug weg ist (0+/-DigitalUebergangsbereichVorgaenger) --> zahl wird gerundet - if ((ergebnis_nachkomma <= 2) || (ergebnis_nachkomma >= 8)) // Band um die Ziffer --> Runden, da Ziffer im Rahmen Ungenauigkeit erreicht + // kein Ziffernwechsel, da Vorgänger weit genug weg ist (0+/-DigitalUebergangsbereichVorgaenger) --> zahl wird gerundet + if ((ergebnis_nachkomma <= DigitalBand) || (ergebnis_nachkomma >= (10-DigitalBand))) // Band um die Ziffer --> Runden, da Ziffer im Rahmen Ungenauigkeit erreicht result = ((int) round(zahl) + 10) % 10; else result = ((int) trunc(zahl) + 10) % 10; @@ -256,6 +245,57 @@ int ClassFlowCNNGeneral::ZeigerEvalHybridNeu(float zahl, float zahl_vorgaenger, } +int ClassFlowCNNGeneral::ZeigerEvalAnalogToDigitNeu(float zahl, float ziffer_vorgaenger) +{ + int result; + int ergebnis_nachkomma = ((int) floor(zahl * 10)) % 10; + int ergebnis_vorkomma = ((int) floor(zahl) + 10) % 10; + + if (ziffer_vorgaenger < 0) + { + result = (int) floor(zahl); + if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalAnalogToDigitNeu - kein Vorgänger - Ergebnis = " + std::to_string(result) + + " zahl: " + std::to_string(zahl) + " ziffer_vorgaenger = " + std::to_string(ziffer_vorgaenger) + " AnalogFehler = " + std::to_string(AnalogFehler)); + return result; + } + + if ((ziffer_vorgaenger >= DigitalUebergangsbereichVorgaengerAnalogToDigit ) && (ziffer_vorgaenger <= (10.0 - DigitalUebergangsbereichVorgaengerAnalogToDigit))) + { + // kein Ziffernwechsel, da Vorgänger weit genug weg ist (0+/-DigitalUebergangsbereichVorgaenger) --> zahl wird gerundet + if ((ergebnis_nachkomma <= 2) || (ergebnis_nachkomma >= 8)) // Band um die Ziffer --> Runden, da Ziffer im Rahmen Ungenauigkeit erreicht + result = ((int) round(zahl) + 10) % 10; + else + result = ((int) trunc(zahl) + 10) % 10; + + if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalAnalogToDigitNeu - kein Ziffernwechsel, da Vorkomma weit genug weg = " + std::to_string(result) + + " zahl: " + std::to_string(zahl) + " ziffer_vorgaenger = " + std::to_string(ziffer_vorgaenger) + " DigitalUnschaerfe = " + std::to_string(DigitalUnschaerfe)); + return result; + } + + if (ziffer_vorgaenger <= 1) // Nulldurchgang hat stattgefunden (!Bewertung über Prev_value und nicht Zahl!) --> hier aufrunden (2.8 --> 3, aber auch 3.1 --> 3) + { + if (ergebnis_nachkomma > 5) + result = (ergebnis_vorkomma + 1) % 10; + else + result = ergebnis_vorkomma; + if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalAnalogToDigitNeu - Nulldurchgang hat stattgefunden = " + std::to_string(result) + + " zahl: " + std::to_string(zahl) + " ziffer_vorgaenger = " + std::to_string(ziffer_vorgaenger) + " DigitalUnschaerfe = " + std::to_string(DigitalUnschaerfe)); + return result; + } + + // bleibt nur >= 9.5 --> noch kein Nulldurchgang --> 2.8 --> 2, und 3.1 --> 2 + // hier auf 4 reduziert, da erst ab Vorgänder 9 anfängt umzustellen. Bei 9.5 Vorgänger kann die aktuelle + // Zahl noch x.4 - x.5 sein. + if (ergebnis_nachkomma >= 4) + result = ergebnis_vorkomma; + else + result = (ergebnis_vorkomma - 1 + 10) % 10; + + if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalAnalogToDigitNeu - 9.0 --> noch kein Nulldurchgang = " + std::to_string(result) + + " zahl: " + std::to_string(zahl) + " ziffer_vorgaenger = " + std::to_string(ziffer_vorgaenger) + " DigitalUnschaerfe = " + std::to_string(DigitalUnschaerfe)); + return result; +} + int ClassFlowCNNGeneral::ZeigerEvalAnalogNeu(float zahl, int ziffer_vorgaenger) { float zahl_min, zahl_max; @@ -281,10 +321,13 @@ int ClassFlowCNNGeneral::ZeigerEvalAnalogNeu(float zahl, int ziffer_vorgaenger) " zahl: " + std::to_string(zahl) + " ziffer_vorgaenger = " + std::to_string(ziffer_vorgaenger) + " AnalogFehler = " + std::to_string(AnalogFehler)); return result; } - result = ((int) floor(zahl_min) + 10) % 10; - if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalAnalogNeu - Zahl uneindeutig, Korrektur nach unten - Ergebnis = " + std::to_string(result) + - " zahl: " + std::to_string(zahl) + " ziffer_vorgaenger = " + std::to_string(ziffer_vorgaenger) + " AnalogFehler = " + std::to_string(AnalogFehler)); - return result; + if (ziffer_vorgaenger >= 10 - AnalogFehler) + { + result = ((int) floor(zahl_min) + 10) % 10; + if (debugdetailgeneral) LogFile.WriteToFile("ClassFlowCNNGeneral::ZeigerEvalAnalogNeu - Zahl uneindeutig, Korrektur nach unten - Ergebnis = " + std::to_string(result) + + " zahl: " + std::to_string(zahl) + " ziffer_vorgaenger = " + std::to_string(ziffer_vorgaenger) + " AnalogFehler = " + std::to_string(AnalogFehler)); + return result; + } } @@ -296,7 +339,7 @@ int ClassFlowCNNGeneral::ZeigerEvalAnalogNeu(float zahl, int ziffer_vorgaenger) } - +/* int ClassFlowCNNGeneral::ZeigerEval(float zahl, int ziffer_vorgaenger) { int ergebnis_nachkomma = ((int) floor(zahl * 10) + 10) % 10; @@ -327,6 +370,7 @@ int ClassFlowCNNGeneral::ZeigerEval(float zahl, int ziffer_vorgaenger) ergebnis = (ergebnis + 10) % 10; return ergebnis; } +*/ bool ClassFlowCNNGeneral::ReadParameter(FILE* pfile, string& aktparamgraph) { diff --git a/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.h b/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.h index 2424ef5f..fd58153c 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.h +++ b/code/components/jomjol_flowcontroll/ClassFlowCNNGeneral.h @@ -25,9 +25,12 @@ protected: std::vector GENERAL; float CNNGoodThreshold; float AnalogFehler = 3.0; + float AnalogToDigtalFehler = 0.8; float DigitalUnschaerfe = 0.2; + int DigitalBand = 3; float DigitalAnalogerVorgaengerUebergangsbereich = 2; - float DigitalUebergangsbereichVorgaenger = 0.7; + float DigitalUebergangsbereichVorgaengerAnalogToDigit = 1; // war vorher 2 + float DigitalUebergangsbereichVorgaenger = 0.9; string cnnmodelfile; int modelxsize, modelysize, modelchannel; @@ -38,9 +41,10 @@ protected: bool SaveAllFiles; // bool extendedResolution; - int ZeigerEval(float zahl, int ziffer_vorgaenger); - int ZeigerEvalHybrid(float zahl, float zahl_vorgaenger, int eval_vorgaenger); +// int ZeigerEval(float zahl, int ziffer_vorgaenger); +// int ZeigerEvalHybrid(float zahl, float zahl_vorgaenger, int eval_vorgaenger); int ZeigerEvalAnalogNeu(float zahl, int ziffer_vorgaenger); + int ZeigerEvalAnalogToDigitNeu(float zahl, float ziffer_vorgaenger); int ZeigerEvalHybridNeu(float zahl, float zahl_vorgaenger, int eval_vorgaenger, bool AnalogerVorgaenger = false); diff --git a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp index 0351dc17..db72aedf 100644 --- a/code/components/jomjol_flowcontroll/ClassFlowControll.cpp +++ b/code/components/jomjol_flowcontroll/ClassFlowControll.cpp @@ -305,6 +305,7 @@ bool ClassFlowControll::doFlow(string time) if (i) i -= 1; // vorheriger Schritt muss wiederholt werden (vermutlich Bilder aufnehmen) result = false; if (repeat > 5) { + LogFile.SwitchOnOff(true); LogFile.WriteToFile("Wiederholung 5x nicht erfolgreich --> reboot"); doReboot(); // Schritt wurde 5x wiederholt --> reboot @@ -493,6 +494,8 @@ bool ClassFlowControll::ReadParameter(FILE* pfile, string& aktparamgraph) // reboot notwendig damit die neue wlan.ini auch benutzt wird !!! fclose(pfile); printf("do reboot\n"); + LogFile.SwitchOnOff(true); + LogFile.WriteToFile("Reboot to activate new HOSTNAME."); esp_restart(); hard_restart(); doReboot(); diff --git a/code/components/jomjol_logfile/ClassLogFile.cpp b/code/components/jomjol_logfile/ClassLogFile.cpp index 04a4df4d..f9a790fe 100644 --- a/code/components/jomjol_logfile/ClassLogFile.cpp +++ b/code/components/jomjol_logfile/ClassLogFile.cpp @@ -73,7 +73,7 @@ void ClassLogFile::WriteToDedicatedFile(std::string _fn, std::string info, bool // pFile = OpenFileAndWait(_fn.c_str(), "a"); pFile = fopen(_fn.c_str(), "a+"); - printf("Logfile opened: %s\n", _fn.c_str()); +// printf("Logfile opened: %s\n", _fn.c_str()); if (pFile!=NULL) { if (_time) diff --git a/code/components/tflite-lib/CMakeLists.txt b/code/components/tflite-lib/CMakeLists.txt index eed31a57..aaf56231 100644 --- a/code/components/tflite-lib/CMakeLists.txt +++ b/code/components/tflite-lib/CMakeLists.txt @@ -25,7 +25,8 @@ list(REMOVE_ITEM srcs_kernels "${tfmicro_kernels_dir}/depthwise_conv.cc" "${tfmicro_kernels_dir}/fully_connected.cc" "${tfmicro_kernels_dir}/mul.cc" - "${tfmicro_kernels_dir}/pooling.cc") + "${tfmicro_kernels_dir}/pooling.cc" + "${tfmicro_kernels_dir}/softmax.cc") FILE(GLOB esp_nn_kernels "${tfmicro_kernels_dir}/esp_nn/*.cc") @@ -38,6 +39,10 @@ set(lib_srcs "${tflite_dir}/kernels/kernel_util.cc" "${tflite_dir}/micro/memory_planner/greedy_memory_planner.cc" "${tflite_dir}/micro/memory_planner/linear_memory_planner.cc" + "${tflite_dir}/micro/arena_allocator/non_persistent_arena_buffer_allocator.cc" + "${tflite_dir}/micro/arena_allocator/persistent_arena_buffer_allocator.cc" + "${tflite_dir}/micro/arena_allocator/recording_single_arena_buffer_allocator.cc" + "${tflite_dir}/micro/arena_allocator/single_arena_buffer_allocator.cc" "${tflite_dir}/c/common.cc" "${tflite_dir}/core/api/error_reporter.cc" "${tflite_dir}/core/api/flatbuffer_conversions.cc" diff --git a/code/components/tflite-lib/tensorflow/lite/builtin_ops.h b/code/components/tflite-lib/tensorflow/lite/builtin_ops.h index 19ce3e2c..01156c39 100644 --- a/code/components/tflite-lib/tensorflow/lite/builtin_ops.h +++ b/code/components/tflite-lib/tensorflow/lite/builtin_ops.h @@ -179,6 +179,12 @@ typedef enum { kTfLiteBuiltinMultinomial = 149, kTfLiteBuiltinGelu = 150, kTfLiteBuiltinDynamicUpdateSlice = 151, + kTfLiteBuiltinRelu0To1 = 152, + kTfLiteBuiltinUnsortedSegmentProd = 153, + kTfLiteBuiltinUnsortedSegmentMax = 154, + kTfLiteBuiltinUnsortedSegmentSum = 155, + kTfLiteBuiltinAtan2 = 156, + kTfLiteBuiltinUnsortedSegmentMin = 157, } TfLiteBuiltinOperator; #ifdef __cplusplus diff --git a/code/components/tflite-lib/tensorflow/lite/c/c_api_types.h b/code/components/tflite-lib/tensorflow/lite/c/c_api_types.h index d2524969..d947213b 100644 --- a/code/components/tflite-lib/tensorflow/lite/c/c_api_types.h +++ b/code/components/tflite-lib/tensorflow/lite/c/c_api_types.h @@ -113,7 +113,13 @@ typedef struct TfLiteQuantizationParams { } TfLiteQuantizationParams; // -------------------------------------------------------------------------- -// Opaque types used by c_api_opaque.h. +// Opaque types used by c_api.h, c_api_opaque.h and common.h. + +// TfLiteOpaqueContext is an opaque version of TfLiteContext; +typedef struct TfLiteOpaqueContext TfLiteOpaqueContext; + +// TfLiteOpaqueNode is an opaque version of TfLiteNode; +typedef struct TfLiteOpaqueNode TfLiteOpaqueNode; // TfLiteOpaqueTensor is an opaque version of TfLiteTensor; typedef struct TfLiteOpaqueTensor TfLiteOpaqueTensor; diff --git a/code/components/tflite-lib/tensorflow/lite/c/common.cc b/code/components/tflite-lib/tensorflow/lite/c/common.cc index 956e9d69..ae5c44b5 100644 --- a/code/components/tflite-lib/tensorflow/lite/c/common.cc +++ b/code/components/tflite-lib/tensorflow/lite/c/common.cc @@ -14,7 +14,11 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/c/common.h" + #include "tensorflow/lite/c/c_api_types.h" +#ifdef TF_LITE_TENSORFLOW_PROFILER +#include "tensorflow/lite/tensorflow_profiler_logger.h" +#endif #ifndef TF_LITE_STATIC_MEMORY #include @@ -99,7 +103,12 @@ void TfLiteFloatArrayFree(TfLiteFloatArray* a) { free(a); } void TfLiteTensorDataFree(TfLiteTensor* t) { if (t->allocation_type == kTfLiteDynamic || t->allocation_type == kTfLitePersistentRo) { - free(t->data.raw); + if (t->data.raw) { +#ifdef TF_LITE_TENSORFLOW_PROFILER + tflite::OnTfLiteTensorDealloc(t); +#endif + free(t->data.raw); + } } t->data.raw = nullptr; } @@ -161,7 +170,7 @@ void TfLiteTensorFree(TfLiteTensor* t) { t->dims = nullptr; if (t->dims_signature) { - TfLiteIntArrayFree((TfLiteIntArray *) t->dims_signature); + TfLiteIntArrayFree((TfLiteIntArray*)t->dims_signature); } t->dims_signature = nullptr; @@ -191,16 +200,12 @@ void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims, } TfLiteStatus TfLiteTensorCopy(const TfLiteTensor* src, TfLiteTensor* dst) { - if (!src || !dst) - return kTfLiteOk; - if (src->bytes != dst->bytes) - return kTfLiteError; - if (src == dst) - return kTfLiteOk; + if (!src || !dst) return kTfLiteOk; + if (src->bytes != dst->bytes) return kTfLiteError; + if (src == dst) return kTfLiteOk; dst->type = src->type; - if (dst->dims) - TfLiteIntArrayFree(dst->dims); + if (dst->dims) TfLiteIntArrayFree(dst->dims); dst->dims = TfLiteIntArrayCopy(src->dims); memcpy(dst->data.raw, src->data.raw, src->bytes); dst->buffer_handle = src->buffer_handle; @@ -218,8 +223,17 @@ void TfLiteTensorRealloc(size_t num_bytes, TfLiteTensor* tensor) { // TODO(b/145340303): Tensor data should be aligned. if (!tensor->data.raw) { tensor->data.raw = (char*)malloc(num_bytes); +#ifdef TF_LITE_TENSORFLOW_PROFILER + tflite::OnTfLiteTensorAlloc(tensor, num_bytes); +#endif } else if (num_bytes > tensor->bytes) { +#ifdef TF_LITE_TENSORFLOW_PROFILER + tflite::OnTfLiteTensorDealloc(tensor); +#endif tensor->data.raw = (char*)realloc(tensor->data.raw, num_bytes); +#ifdef TF_LITE_TENSORFLOW_PROFILER + tflite::OnTfLiteTensorAlloc(tensor, num_bytes); +#endif } tensor->bytes = num_bytes; } diff --git a/code/components/tflite-lib/tensorflow/lite/c/common.h b/code/components/tflite-lib/tensorflow/lite/c/common.h index 6a109e1e..cc856f9a 100644 --- a/code/components/tflite-lib/tensorflow/lite/c/common.h +++ b/code/components/tflite-lib/tensorflow/lite/c/common.h @@ -173,9 +173,9 @@ void TfLiteFloatArrayFree(TfLiteFloatArray* a); } \ } while (false) #else // TF_LITE_STRIP_ERROR_STRINGS -#define UNUSED(...) (void)sizeof(#__VA_ARGS__) -#define TF_LITE_KERNEL_LOG(context, ...) UNUSED(__VA_ARGS__) -#define TF_LITE_MAYBE_KERNEL_LOG(context, ...) UNUSED(__VA_ARGS__) +#define ARGS_UNUSED(...) (void)sizeof(#__VA_ARGS__) +#define TF_LITE_KERNEL_LOG(context, ...) ARGS_UNUSED(__VA_ARGS__) +#define TF_LITE_MAYBE_KERNEL_LOG(context, ...) ARGS_UNUSED(__VA_ARGS__) #endif // TF_LITE_STRIP_ERROR_STRINGS // Check whether value is true, and if not return kTfLiteError from @@ -842,6 +842,12 @@ typedef struct TfLiteContext { size_t* bytes); } TfLiteContext; +// `TfLiteRegistrationExternal` is an external version of `TfLiteRegistration` +// for C API which doesn't use internal types (such as `TfLiteContext`) but only +// uses stable API types (such as `TfLiteOpaqueContext`). The purpose of each +// field is the exactly the same as with `TfLiteRegistration`. +typedef struct TfLiteRegistrationExternal TfLiteRegistrationExternal; + typedef struct TfLiteRegistration { // Initializes the op from serialized data. // Called only *once* for the lifetime of the op, so any one-time allocations @@ -903,8 +909,31 @@ typedef struct TfLiteRegistration { // Note: It is the responsibility of the registration binder to set this // properly. int version; + + // The external version of `TfLiteRegistration`. Since we can't use internal + // types (such as `TfLiteContext`) for C API to maintain ABI stability. + // C API user will provide `TfLiteRegistrationExternal` to implement custom + // ops. We keep it inside of `TfLiteRegistration` and use it to route + // callbacks properly. + TfLiteRegistrationExternal* registration_external; } TfLiteRegistration; +// Old version of `TfLiteRegistration` to maintain binary backward +// compatibility. +// WARNING: This structure is deprecated / not an official part of the API. +// It should be only used for binary backward compatibility. +typedef struct TfLiteRegistration_V1 { + void* (*init)(TfLiteContext* context, const char* buffer, size_t length); + void (*free)(TfLiteContext* context, void* buffer); + TfLiteStatus (*prepare)(TfLiteContext* context, TfLiteNode* node); + TfLiteStatus (*invoke)(TfLiteContext* context, TfLiteNode* node); + const char* (*profiling_string)(const TfLiteContext* context, + const TfLiteNode* node); + int32_t builtin_code; + const char* custom_name; + int version; +} TfLiteRegistration_V1; + // The flags used in `TfLiteDelegate`. Note that this is a bitmask, so the // values should be 1, 2, 4, 8, ...etc. typedef enum TfLiteDelegateFlags { diff --git a/code/components/tflite-lib/tensorflow/lite/core/api/flatbuffer_conversions.cc b/code/components/tflite-lib/tensorflow/lite/core/api/flatbuffer_conversions.cc index e92d754f..1ecefa47 100644 --- a/code/components/tflite-lib/tensorflow/lite/core/api/flatbuffer_conversions.cc +++ b/code/components/tflite-lib/tensorflow/lite/core/api/flatbuffer_conversions.cc @@ -493,6 +493,11 @@ TfLiteStatus ParseOpDataTfLite(const Operator* op, BuiltinOperator op_type, return ParseSquare(op, error_reporter, allocator, builtin_data); } + case BuiltinOperator_SQUARED_DIFFERENCE: { + return ParseSquaredDifference(op, error_reporter, allocator, + builtin_data); + } + case BuiltinOperator_SQUEEZE: { return ParseSqueeze(op, error_reporter, allocator, builtin_data); } @@ -840,14 +845,25 @@ TfLiteStatus ParseOpDataTfLite(const Operator* op, BuiltinOperator op_type, // TODO(aselle): Implement call in BuiltinOptions, but nullptrs are // ok for now, since there is no call implementation either. case BuiltinOperator_CALL: + case BuiltinOperator_COMPLEX_ABS: case BuiltinOperator_CONCAT_EMBEDDINGS: case BuiltinOperator_COS: case BuiltinOperator_CUSTOM: + case BuiltinOperator_DENSIFY: + case BuiltinOperator_DYNAMIC_UPDATE_SLICE: case BuiltinOperator_EMBEDDING_LOOKUP: case BuiltinOperator_EQUAL: + case BuiltinOperator_HASHTABLE_FIND: + case BuiltinOperator_HASHTABLE_IMPORT: + case BuiltinOperator_HASHTABLE_SIZE: + case BuiltinOperator_IMAG: case BuiltinOperator_MATRIX_DIAG: case BuiltinOperator_MATRIX_SET_DIAG: + case BuiltinOperator_NON_MAX_SUPPRESSION_V4: + case BuiltinOperator_NON_MAX_SUPPRESSION_V5: case BuiltinOperator_RELU_N1_TO_1: + case BuiltinOperator_RELU_0_TO_1: + case BuiltinOperator_SCATTER_ND: case BuiltinOperator_SELECT: case BuiltinOperator_SELECT_V2: case BuiltinOperator_SLICE: @@ -855,23 +871,17 @@ TfLiteStatus ParseOpDataTfLite(const Operator* op, BuiltinOperator op_type, case BuiltinOperator_TOPK_V2: case BuiltinOperator_TRANSPOSE: case BuiltinOperator_RANGE: - case BuiltinOperator_SQUARED_DIFFERENCE: - case BuiltinOperator_REVERSE_V2: - case BuiltinOperator_WHERE: case BuiltinOperator_RANK: - case BuiltinOperator_NON_MAX_SUPPRESSION_V4: - case BuiltinOperator_NON_MAX_SUPPRESSION_V5: - case BuiltinOperator_SCATTER_ND: - case BuiltinOperator_DENSIFY: - case BuiltinOperator_SEGMENT_SUM: - case BuiltinOperator_RFFT2D: - case BuiltinOperator_IMAG: case BuiltinOperator_REAL: - case BuiltinOperator_COMPLEX_ABS: - case BuiltinOperator_HASHTABLE_FIND: - case BuiltinOperator_HASHTABLE_IMPORT: - case BuiltinOperator_HASHTABLE_SIZE: - case BuiltinOperator_DYNAMIC_UPDATE_SLICE: + case BuiltinOperator_RFFT2D: + case BuiltinOperator_SEGMENT_SUM: + case BuiltinOperator_REVERSE_V2: + case BuiltinOperator_UNSORTED_SEGMENT_MAX: + case BuiltinOperator_UNSORTED_SEGMENT_MIN: + case BuiltinOperator_UNSORTED_SEGMENT_PROD: + case BuiltinOperator_UNSORTED_SEGMENT_SUM: + case BuiltinOperator_ATAN2: + case BuiltinOperator_WHERE: return kTfLiteOk; case BuiltinOperator_PLACEHOLDER_FOR_GREATER_OP_CODES: return kTfLiteError; @@ -2189,6 +2199,14 @@ TfLiteStatus ParseSquare(const Operator*, ErrorReporter*, BuiltinDataAllocator*, return kTfLiteOk; } +// We have this parse function instead of directly returning kTfLiteOk from the +// switch-case in ParseOpData because this function is used as part of the +// selective registration for the OpResolver implementation in micro. +TfLiteStatus ParseSquaredDifference(const Operator*, ErrorReporter*, + BuiltinDataAllocator*, void**) { + return kTfLiteOk; +} + TfLiteStatus ParseStridedSlice(const Operator* op, ErrorReporter* error_reporter, BuiltinDataAllocator* allocator, diff --git a/code/components/tflite-lib/tensorflow/lite/core/api/flatbuffer_conversions.h b/code/components/tflite-lib/tensorflow/lite/core/api/flatbuffer_conversions.h index cd6637bc..ed317b81 100644 --- a/code/components/tflite-lib/tensorflow/lite/core/api/flatbuffer_conversions.h +++ b/code/components/tflite-lib/tensorflow/lite/core/api/flatbuffer_conversions.h @@ -356,6 +356,11 @@ TfLiteStatus ParseSqrt(const Operator* op, ErrorReporter* error_reporter, TfLiteStatus ParseSquare(const Operator* op, ErrorReporter* error_reporter, BuiltinDataAllocator* allocator, void** builtin_data); +TfLiteStatus ParseSquaredDifference(const Operator* op, + ErrorReporter* error_reporter, + BuiltinDataAllocator* allocator, + void** builtin_data); + TfLiteStatus ParseStridedSlice(const Operator* op, ErrorReporter* error_reporter, BuiltinDataAllocator* allocator, diff --git a/code/components/tflite-lib/tensorflow/lite/core/api/op_resolver.h b/code/components/tflite-lib/tensorflow/lite/core/api/op_resolver.h index 49ac778e..cec1f2dd 100644 --- a/code/components/tflite-lib/tensorflow/lite/core/api/op_resolver.h +++ b/code/components/tflite-lib/tensorflow/lite/core/api/op_resolver.h @@ -23,6 +23,16 @@ limitations under the License. #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/schema/schema_generated.h" +// Opaque type similar to TfLiteDelegate / TfLiteOpaqueDelegate. +// This is used for cases (e.g. when using "TF Lite with Google Play Services") +// where the TF Lite runtime might be built using a newer (or older) +// version of the TF Lite sources than the app, and hence might have a +// different definition of the TfLiteDelegate type. TF Lite APIs use +// TfLiteOpaqueDelegate rather than TfLiteDelegate when they want to +// refer to a delegate defined with that potentially different version +// of the TfLiteDelegate type. +struct TfLiteOpaqueDelegateStruct; + namespace tflite { /// Abstract interface that returns TfLiteRegistrations given op codes or custom @@ -37,8 +47,10 @@ class OpResolver { virtual const TfLiteRegistration* FindOp(const char* op, int version) const = 0; + // Represents a sequence of delegates. using TfLiteDelegatePtrVector = std::vector>; + // Returns optional delegates for resolving and handling ops in the flatbuffer // model. This may be used in addition to the standard TfLiteRegistration // lookup for graph resolution. @@ -47,16 +59,55 @@ class OpResolver { return {}; } - // Represent a function that creates a TfLite delegate instance. + // Represents a function that creates a TfLite delegate instance. using TfLiteDelegateCreator = std::function( int /*num_threads*/)>; + + // Represents a sequence of delegate creator functions. using TfLiteDelegateCreators = std::vector; + // Returns a vector of delegate creators to create optional delegates for // resolving and handling ops in the flatbuffer model. This may be used in // addition to the standard TfLiteRegistration lookup for graph resolution. + // + // Note that this method is not used (will not be called) if you are using + // TF Lite in Google Play Services; the GetOpaqueDelegateCreators method + // (see below) is used for that case. virtual TfLiteDelegateCreators GetDelegateCreators() const { return {}; } + // TODO(b/202712825): it would be nice if we could avoid the need for separate + // "opaque" types & methods for use only with TF Lite in Google Play Services. + + // Represents an opaque delegate instance. + // WARNING: Experimental interface, subject to change. + using TfLiteOpaqueDelegatePtr = + std::unique_ptr; + + // Represents a function that creates an opaque delegate instance. + // WARNING: Experimental interface, subject to change. + using TfLiteOpaqueDelegateCreator = + std::function; + + // Represents a sequence of opaque delegate creator functions. + // WARNING: Experimental interface, subject to change. + using TfLiteOpaqueDelegateCreators = std::vector; + + // Returns a vector of opaque delegate creators to create optional opaque + // delegates for resolving and handling ops in the flatbuffer model. This may + // be used in addition to the standard TfLiteRegistration lookup for graph + // resolution. + // + // Note that this method will be called only if you are using TF Lite in + // Google Play Services; if you are using regular TF Lite, GetDelegateCreators + // (see above) is used instead. + // + // WARNING: Experimental interface, subject to change. + virtual TfLiteOpaqueDelegateCreators GetOpaqueDelegateCreators() const { + return {}; + } + virtual ~OpResolver() {} private: diff --git a/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/fft.cc b/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/fft.cc index 62442fba..bcdd9cc0 100644 --- a/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/fft.cc +++ b/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/fft.cc @@ -13,10 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ #include "tensorflow/lite/experimental/microfrontend/lib/fft.h" -#include "tensorflow/lite/experimental/microfrontend/lib/kiss_fft_int16.h" #include +#include "tensorflow/lite/experimental/microfrontend/lib/kiss_fft_int16.h" void FftCompute(struct FftState* state, const int16_t* input, int input_scale_shift) { @@ -37,9 +37,9 @@ void FftCompute(struct FftState* state, const int16_t* input, // Apply the FFT. kissfft_fixed16::kiss_fftr( - reinterpret_cast(state->scratch), - state->input, - reinterpret_cast(state->output)); + reinterpret_cast(state->scratch), + state->input, + reinterpret_cast(state->output)); } void FftInit(struct FftState* state) { diff --git a/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/fft_util.cc b/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/fft_util.cc index 81efe14d..ed3dc8fb 100644 --- a/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/fft_util.cc +++ b/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/fft_util.cc @@ -13,10 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ #include "tensorflow/lite/experimental/microfrontend/lib/fft_util.h" -#include "tensorflow/lite/experimental/microfrontend/lib/kiss_fft_int16.h" #include +#include "tensorflow/lite/experimental/microfrontend/lib/kiss_fft_int16.h" + int FftPopulateState(struct FftState* state, size_t input_size) { state->input_size = input_size; state->fft_size = 1; diff --git a/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/kiss_fft_int16.h b/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/kiss_fft_int16.h index 9abe686b..beee99aa 100644 --- a/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/kiss_fft_int16.h +++ b/code/components/tflite-lib/tensorflow/lite/experimental/microfrontend/lib/kiss_fft_int16.h @@ -31,4 +31,3 @@ namespace kissfft_fixed16 { #undef KISS_FFT_H #endif // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_KISS_FFT_INT16_H_ - diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/common.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/common.h index 5e8778f1..205294fd 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/common.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/common.h @@ -15,6 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_COMMON_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_COMMON_H_ +#include #ifndef ALLOW_SLOW_GENERIC_DEPTHWISECONV_FALLBACK #ifdef GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK #define ALLOW_SLOW_GENERIC_DEPTHWISECONV_FALLBACK diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/compatibility.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/compatibility.h index 61becad3..7ba66ed8 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/compatibility.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/compatibility.h @@ -86,6 +86,16 @@ using int32 = std::int32_t; using uint32 = std::uint32_t; #endif // !defined(TF_LITE_STATIC_MEMORY) +// Allow for cross-compiler usage of function signatures - currently used for +// specifying named RUY profiler regions in templated methods. +#if defined(_MSC_VER) +#define TFLITE_PRETTY_FUNCTION __FUNCSIG__ +#elif defined(__GNUC__) +#define TFLITE_PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define TFLITE_PRETTY_FUNCTION __func__ +#endif + // TFLITE_DEPRECATED() // // Duplicated from absl/base/macros.h to avoid pulling in that library. diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/portable_tensor_utils.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/portable_tensor_utils.h index ab0c8f96..122a0dc2 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/portable_tensor_utils.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/portable_tensor_utils.h @@ -324,7 +324,7 @@ void ApplySigmoidFloat(const int16_t* input, int32_t n_batch, int32_t n_input, // - n_input: the size for input and output. // - output: the 16 bit output // The input is in Qm.15-m format and the output is in Q0.15 format. -void ApplyTanh(int32_t integer_bits, const int16_t* input, int32_t n_batch, +void ApplyTanh(int32_t intger_bits, const int16_t* input, int32_t n_batch, int32_t n_input, int16_t* output); // Apply Tanh to a quantized vector. Tbe internal calculation is in float. diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/add.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/add.h index 57fa13d8..1f521316 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/add.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/add.h @@ -15,6 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_ADD_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_ADD_H_ +#include #include #include "fixedpoint/fixedpoint.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/concatenation.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/concatenation.h index 998bb093..9d2ecbec 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/concatenation.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/concatenation.h @@ -16,6 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_CONCATENATION_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_CONCATENATION_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/internal/cppmath.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/conv.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/conv.h index ac5f04f6..3a53e06e 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/conv.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/conv.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_CONV_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_CONV_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/types.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/div.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/div.h new file mode 100644 index 00000000..df8da1b1 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/div.h @@ -0,0 +1,247 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_DIV_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_DIV_H_ + +#include + +#include "tensorflow/lite/kernels/internal/common.h" + +namespace tflite { + +namespace reference_ops { + +template +inline void DivCheckArithmeticParams(const ArithmeticParams& params) { + TFLITE_DCHECK_LE(params.quantized_activation_min, + params.quantized_activation_max); + // Input offset is negative input zero point. Activation tensors are + // asymmetric quantized so they span the full int8 range. + constexpr int32_t max_value = + static_cast(std::numeric_limits::max()); + TFLITE_DCHECK_GE(params.input1_offset, -max_value); + TFLITE_DCHECK_LE(params.input1_offset, max_value); + TFLITE_DCHECK_GE(params.input2_offset, -max_value); + TFLITE_DCHECK_LE(params.input2_offset, max_value); + TFLITE_DCHECK_GE(params.output_offset, -max_value); + TFLITE_DCHECK_LE(params.output_offset, max_value); +} + +// Element-wise div that can often be used for inner loop of broadcast Div as +// well as the non-broadcast Div. +template +inline void DivElementwise(int size, const ArithmeticParams& params, + const T* input1_data, const T* input2_data, + T* output_data) { + DivCheckArithmeticParams(params); + + for (int i = 0; i < size; ++i) { + int32_t input1_val = params.input1_offset + input1_data[i]; + int32_t input2_val = params.input2_offset + input2_data[i]; + TFLITE_DCHECK_NE(input2_val, 0); + if (input2_val < 0) { + // Invert signs to avoid a negative input2_val as input2_inv needs to be + // positive to be used as multiplier of MultiplyByQuantizedMultiplier. + input1_val = -input1_val; + input2_val = -input2_val; + } + int recip_shift; + const int32_t input2_inv = GetReciprocal(input2_val, 31, &recip_shift); + const int headroom = CountLeadingSignBits(input1_val); + const int32_t unscaled_quotient = + MultiplyByQuantizedMultiplierGreaterThanOne(input1_val, input2_inv, + headroom); + const int total_shift = params.output_shift - recip_shift - headroom; + const int32_t unclamped_result = + params.output_offset + + MultiplyByQuantizedMultiplierSmallerThanOneExp( + unscaled_quotient, params.output_multiplier, total_shift); + const int32_t clamped_output = + std::min(params.quantized_activation_max, + std::max(params.quantized_activation_min, unclamped_result)); + output_data[i] = static_cast(clamped_output); + } +} + +inline void Div(const ArithmeticParams& params, + const RuntimeShape& input1_shape, const uint8_t* input1_data, + const RuntimeShape& input2_shape, const uint8_t* input2_data, + const RuntimeShape& output_shape, uint8_t* output_data) { + TFLITE_DCHECK_LE(params.quantized_activation_min, + params.quantized_activation_max); + const int flat_size = + MatchingElementsSize(input1_shape, input2_shape, output_shape); + + DivElementwise(flat_size, params, input1_data, input2_data, output_data); +} + +inline void Div(const ArithmeticParams& params, + const RuntimeShape& input1_shape, const int8_t* input1_data, + const RuntimeShape& input2_shape, const int8_t* input2_data, + const RuntimeShape& output_shape, int8_t* output_data) { + TFLITE_DCHECK_LE(params.quantized_activation_min, + params.quantized_activation_max); + const int flat_size = + MatchingElementsSize(input1_shape, input2_shape, output_shape); + + DivElementwise(flat_size, params, input1_data, input2_data, output_data); +} + +template +inline void BroadcastDivSlowQuantized( + const ArithmeticParams& params, const RuntimeShape& unextended_input1_shape, + const T* input1_data, const RuntimeShape& unextended_input2_shape, + const T* input2_data, const RuntimeShape& unextended_output_shape, + T* output_data) { + TFLITE_DCHECK_LE(unextended_input1_shape.DimensionsCount(), N); + TFLITE_DCHECK_LE(unextended_input2_shape.DimensionsCount(), N); + TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), N); + + NdArrayDesc desc1; + NdArrayDesc desc2; + NdArrayDesc output_desc; + NdArrayDescsForElementwiseBroadcast(unextended_input1_shape, + unextended_input2_shape, &desc1, &desc2); + CopyDimsToDesc(RuntimeShape::ExtendedShape(N, unextended_output_shape), + &output_desc); + + DivCheckArithmeticParams(params); + + auto div_func = [&](int indexes[N]) { + int32_t input1_val = + params.input1_offset + input1_data[SubscriptToIndex(desc1, indexes)]; + int32_t input2_val = + params.input2_offset + input2_data[SubscriptToIndex(desc2, indexes)]; + TFLITE_DCHECK_NE(input2_val, 0); + if (input2_val < 0) { + // Invert signs to avoid a negative input2_val as input2_inv needs to be + // positive to be used as multiplier of MultiplyByQuantizedMultiplier. + input1_val = -input1_val; + input2_val = -input2_val; + } + int recip_shift; + const int32_t input2_inv = GetReciprocal(input2_val, 31, &recip_shift); + const int headroom = CountLeadingSignBits(input1_val); + const int32_t unscaled_quotient = + MultiplyByQuantizedMultiplierGreaterThanOne(input1_val, input2_inv, + headroom); + const int total_shift = params.output_shift - recip_shift - headroom; + const int32_t unclamped_result = + params.output_offset + + MultiplyByQuantizedMultiplierSmallerThanOneExp( + unscaled_quotient, params.output_multiplier, total_shift); + const int32_t clamped_output = + std::min(params.quantized_activation_max, + std::max(params.quantized_activation_min, unclamped_result)); + output_data[SubscriptToIndex(output_desc, indexes)] = + static_cast(clamped_output); + }; + NDOpsHelper(output_desc, div_func); +} + +template +inline void BroadcastDivSlow(const ArithmeticParams& params, + const RuntimeShape& unextended_input1_shape, + const uint8_t* input1_data, + const RuntimeShape& unextended_input2_shape, + const uint8_t* input2_data, + const RuntimeShape& unextended_output_shape, + uint8_t* output_data) { + BroadcastDivSlowQuantized( + params, unextended_input1_shape, input1_data, unextended_input2_shape, + input2_data, unextended_output_shape, output_data); +} + +template +inline void BroadcastDivSlow(const ArithmeticParams& params, + const RuntimeShape& unextended_input1_shape, + const int8_t* input1_data, + const RuntimeShape& unextended_input2_shape, + const int8_t* input2_data, + const RuntimeShape& unextended_output_shape, + int8_t* output_data) { + BroadcastDivSlowQuantized( + params, unextended_input1_shape, input1_data, unextended_input2_shape, + input2_data, unextended_output_shape, output_data); +} + +// TODO(jiawen): We can implement BroadcastDiv on buffers of arbitrary +// dimensionality if the runtime code does a single loop over one dimension +// that handles broadcasting as the base case. The code generator would then +// generate max(D1, D2) nested for loops. +template +void BroadcastDivSlow(const ArithmeticParams& params, + const RuntimeShape& unextended_input1_shape, + const T* input1_data, + const RuntimeShape& unextended_input2_shape, + const T* input2_data, + const RuntimeShape& unextended_output_shape, + T* output_data) { + T output_activation_min; + T output_activation_max; + GetActivationParams(params, &output_activation_min, &output_activation_max); + + TFLITE_DCHECK_LE(unextended_input1_shape.DimensionsCount(), N); + TFLITE_DCHECK_LE(unextended_input2_shape.DimensionsCount(), N); + TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), N); + + NdArrayDesc desc1; + NdArrayDesc desc2; + NdArrayDesc output_desc; + NdArrayDescsForElementwiseBroadcast(unextended_input1_shape, + unextended_input2_shape, &desc1, &desc2); + CopyDimsToDesc(RuntimeShape::ExtendedShape(N, unextended_output_shape), + &output_desc); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest + // stride, typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + + auto div_func = [&](int indexes[N]) { + output_data[SubscriptToIndex(output_desc, indexes)] = + ActivationFunctionWithMinMax( + input1_data[SubscriptToIndex(desc1, indexes)] / + input2_data[SubscriptToIndex(desc2, indexes)], + output_activation_min, output_activation_max); + }; + NDOpsHelper(output_desc, div_func); +} + +template +inline void Div(const ArithmeticParams& params, + const RuntimeShape& input1_shape, const T* input1_data, + const RuntimeShape& input2_shape, const T* input2_data, + const RuntimeShape& output_shape, T* output_data) { + T output_activation_min; + T output_activation_max; + GetActivationParams(params, &output_activation_min, &output_activation_max); + + const int flat_size = + MatchingElementsSize(input1_shape, input2_shape, output_shape); + for (int i = 0; i < flat_size; ++i) { + output_data[i] = ActivationFunctionWithMinMax( + input1_data[i] / input2_data[i], output_activation_min, + output_activation_max); + } +} + +} // namespace reference_ops +} // namespace tflite + +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_DIV_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/fully_connected.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/fully_connected.h index 9bf2e5df..ba51cbcf 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/fully_connected.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/fully_connected.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_FULLY_CONNECTED_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_FULLY_CONNECTED_H_ +#include + #include "ruy/profiler/instrumentation.h" // from @ruy #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/cppmath.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/hard_swish.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/hard_swish.h index cda1b5cf..d9fe32e9 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/hard_swish.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/hard_swish.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_ACTIVATIONS_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_ACTIVATIONS_H_ +#include + #include "ruy/profiler/instrumentation.h" // from @ruy #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/types.h" @@ -23,9 +25,9 @@ namespace tflite { namespace reference_ops { inline int16_t SaturatingLeftShift(int16_t value, int amount) { - int32_t result = static_cast(value) * (1 << amount); - result = std::min(result, std::numeric_limits::max()); - result = std::max(result, std::numeric_limits::min()); + int64_t result = static_cast(value) * (1 << amount); + result = std::min(result, std::numeric_limits::max()); + result = std::max(result, std::numeric_limits::min()); return result; } diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/add.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/add.h index 10bee904..8d9b318c 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/add.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/add.h @@ -15,6 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_ADD_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_ADD_H_ +#include #include #include "tensorflow/lite/kernels/internal/common.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h index 3f869a3a..ffd4978e 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_CONV_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_CONV_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h index f0ca09c7..7676fce0 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_DEPTHWISE_CONV_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_DEPTHWISE_CONV_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h index 42920d16..634f0bff 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h @@ -15,11 +15,101 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_FULLY_CONNECTED_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_FULLY_CONNECTED_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { namespace reference_integer_ops { +// For per-channel functions, since it is defined in quantization spec that +// weights are symmetric +// (https://www.tensorflow.org/lite/performance/quantization_spec#symmetric_vs_asymmetric), +// zero_point (params.weights_offset) is always 0. +// However, for per-tensor functions, params.weights_offset is still applied for +// backward compatibility. + +inline void FullyConnectedPerChannel( + const FullyConnectedParams& params, const int32_t* output_multiplier, + const int* output_shift, const RuntimeShape& input_shape, + const int8_t* input_data, const RuntimeShape& filter_shape, + const int8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + int8_t* output_data) { + const int32_t input_offset = params.input_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; + TFLITE_DCHECK_GE(filter_shape.DimensionsCount(), 2); + TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 2); + + TFLITE_DCHECK_LE(output_activation_min, output_activation_max); + const int filter_dim_count = filter_shape.DimensionsCount(); + const int batches = output_shape.Dims(0); + const int output_depth = output_shape.Dims(1); + TFLITE_DCHECK_LE(output_depth, filter_shape.Dims(filter_dim_count - 2)); + const int accum_depth = filter_shape.Dims(filter_dim_count - 1); + for (int b = 0; b < batches; ++b) { + for (int out_c = 0; out_c < output_depth; ++out_c) { + int32_t acc = 0; + for (int d = 0; d < accum_depth; ++d) { + int32_t input_val = input_data[b * accum_depth + d]; + int32_t filter_val = filter_data[out_c * accum_depth + d]; + acc += filter_val * (input_val + input_offset); + } + if (bias_data) { + acc += bias_data[out_c]; + } + acc = MultiplyByQuantizedMultiplier(acc, output_multiplier[out_c], + output_shift[out_c]); + acc += output_offset; + acc = std::max(acc, output_activation_min); + acc = std::min(acc, output_activation_max); + output_data[out_c + output_depth * b] = static_cast(acc); + } + } +} + +template +inline void FullyConnectedPerChannel( + const FullyConnectedParams& params, const int32_t* output_multiplier, + const int* output_shift, const RuntimeShape& input_shape, + const int16_t* input_data, const RuntimeShape& filter_shape, + const int8_t* filter_data, const RuntimeShape& bias_shape, + const AccumScalar* bias_data, const RuntimeShape& output_shape, + int16_t* output_data) { + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; + TFLITE_DCHECK_GE(filter_shape.DimensionsCount(), 2); + TFLITE_DCHECK_GE(output_shape.DimensionsCount(), 1); + + TFLITE_DCHECK_LE(output_activation_min, output_activation_max); + const int filter_dim_count = filter_shape.DimensionsCount(); + const int output_dim_count = output_shape.DimensionsCount(); + const int batches = FlatSizeSkipDim(output_shape, output_dim_count - 1); + const int output_depth = output_shape.Dims(output_dim_count - 1); + TFLITE_DCHECK_LE(output_depth, filter_shape.Dims(filter_dim_count - 2)); + const int accum_depth = filter_shape.Dims(filter_dim_count - 1); + for (int b = 0; b < batches; ++b) { + for (int out_c = 0; out_c < output_depth; ++out_c) { + AccumScalar acc = 0; + for (int d = 0; d < accum_depth; ++d) { + int32_t input_val = input_data[b * accum_depth + d]; + int32_t filter_val = filter_data[out_c * accum_depth + d]; + acc += filter_val * input_val; + } + if (bias_data) { + acc += bias_data[out_c]; + } + int32_t acc_scaled = MultiplyByQuantizedMultiplier( + acc, output_multiplier[out_c], output_shift[out_c]); + acc_scaled = std::max(acc_scaled, output_activation_min); + acc_scaled = std::min(acc_scaled, output_activation_max); + output_data[out_c + output_depth * b] = static_cast(acc_scaled); + } + } +} + inline void FullyConnected( const FullyConnectedParams& params, const RuntimeShape& input_shape, const int8_t* input_data, const RuntimeShape& filter_shape, diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h index 31f2de98..164a8367 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_L2NORMALIZATION_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_L2NORMALIZATION_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h index 95697ec9..16eff133 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h @@ -15,7 +15,9 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_LOGISTIC_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_LOGISTIC_H_ +#include #include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/mean.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/mean.h index bd484270..09d37b72 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/mean.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/mean.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_MEAN_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_MEAN_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/mul.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/mul.h index b80838aa..22e89740 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/mul.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/mul.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_MUL_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_MUL_H_ +#include + #include "fixedpoint/fixedpoint.h" #include "ruy/profiler/instrumentation.h" // from @ruy #include "tensorflow/lite/kernels/internal/common.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h index 2cb4dada..4dc31d9e 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h @@ -15,7 +15,9 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_POOLING_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_POOLING_H_ +#include #include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h index 63e40936..7b1e003b 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h @@ -15,6 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_TANH_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_TANH_H_ +#include #include #include "fixedpoint/fixedpoint.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/transpose_conv.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/transpose_conv.h index 3397f869..92919a71 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/transpose_conv.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/integer_ops/transpose_conv.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_TRANSPOSE_CONV_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_TRANSPOSE_CONV_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/mul.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/mul.h index 273a8a98..b977104c 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/mul.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/mul.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_MUL_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_MUL_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/pooling.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/pooling.h index ee30b840..fe17484c 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/pooling.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/pooling.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_POOLING_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_POOLING_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/cppmath.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/prelu.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/prelu.h index 02db5174..aa9901d6 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/prelu.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/prelu.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_PRELU_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_PRELU_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/internal/types.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h index 40f779c5..bda27693 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_PROCESS_BROADCAST_SHAPES_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_PROCESS_BROADCAST_SHAPES_H_ +#include + #include "tensorflow/lite/kernels/internal/types.h" namespace tflite { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/reduce.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/reduce.h index 348e170e..341b3a08 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/reduce.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/reduce.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_REDUCE_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_REDUCE_H_ +#include + #include "ruy/profiler/instrumentation.h" // from @ruy #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/cppmath.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/requantize.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/requantize.h index d1e67785..f35f6fc8 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/requantize.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/requantize.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_REQUANTIZE_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_REQUANTIZE_H_ +#include + #include "ruy/profiler/instrumentation.h" // from @ruy #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/types.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h index 95550abc..bf0b757e 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h @@ -15,6 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_RESIZE_NEAREST_NEIGHBOR_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_RESIZE_NEAREST_NEIGHBOR_H_ +#include #include #include "tensorflow/lite/kernels/internal/cppmath.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/softmax.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/softmax.h index c2bddcf7..9f4b6398 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/softmax.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/softmax.h @@ -15,6 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_SOFTMAX_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_SOFTMAX_H_ +#include #include #include "fixedpoint/fixedpoint.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/transpose_conv.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/transpose_conv.h index 6e9cb1f9..ac91f379 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/transpose_conv.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/reference/transpose_conv.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_TRANSPOSE_CONV_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_TRANSPOSE_CONV_H_ +#include + #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/types.h" diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/runtime_shape.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/runtime_shape.h index 13693643..c2678b57 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/runtime_shape.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/runtime_shape.h @@ -27,6 +27,11 @@ class RuntimeShape { public: RuntimeShape& operator=(RuntimeShape const&) = delete; + // RuntimeShape in TFLM supports up to 5 dimensions. + // The name kMaxSmallSize comes from the same file of the upstream + // tensorflow lite repo and need to be kept the same for max reuse. + static constexpr int kMaxSmallSize = 5; + RuntimeShape() : size_(0) {} explicit RuntimeShape(int dimensions_count) : size_(dimensions_count) {} @@ -104,11 +109,9 @@ class RuntimeShape { sizeof(int32_t) * shape.DimensionsCount()); } - // A maximum of 4 dimensions are supported on TFLM. - static constexpr int kMaxSize = 5; int32_t size_; union { - int32_t dims_[kMaxSize]; + int32_t dims_[kMaxSmallSize]; }; }; diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/internal/types.h b/code/components/tflite-lib/tensorflow/lite/kernels/internal/types.h index 77644bc0..c44ba48e 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/internal/types.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/internal/types.h @@ -974,11 +974,11 @@ struct StridedSliceParams { int8_t strides_count; int32_t strides[5]; - int16_t begin_mask; - int16_t ellipsis_mask; - int16_t end_mask; - int16_t new_axis_mask; - int16_t shrink_axis_mask; + uint16_t begin_mask; + uint16_t ellipsis_mask; + uint16_t end_mask; + uint16_t new_axis_mask; + uint16_t shrink_axis_mask; }; struct TanhParams { diff --git a/code/components/tflite-lib/tensorflow/lite/kernels/kernel_util.h b/code/components/tflite-lib/tensorflow/lite/kernels/kernel_util.h index 22689436..06874422 100644 --- a/code/components/tflite-lib/tensorflow/lite/kernels/kernel_util.h +++ b/code/components/tflite-lib/tensorflow/lite/kernels/kernel_util.h @@ -177,6 +177,14 @@ inline int64_t NumElements(const TfLiteTensor* t) { return NumElements(t->dims); } +inline int64_t NumElements(const int* dims, int num_dims) { + int64_t count = 1; + for (int i = 0; i < num_dims; ++i) { + count *= dims[i]; + } + return count; +} + // Determines whether tensor is constant. // TODO(b/138199592): Introduce new query which checks for constant OR // persistent-read-only, which would be useful for most tensor kernels that @@ -308,7 +316,7 @@ TfLiteStatus CalculateShapeForBroadcast(TfLiteContext* context, const TfLiteTensor* input3, TfLiteIntArray** output_shape); -// Return the size of given type in bytes. Return 0 in in case of string. +// Return the size of given type in bytes. Return 0 in case of string. int TfLiteTypeGetSize(TfLiteType type); // Whether the current platform is mobile (Android or iOS). diff --git a/code/components/tflite-lib/tensorflow/lite/micro/all_ops_resolver.cc b/code/components/tflite-lib/tensorflow/lite/micro/all_ops_resolver.cc index 6fa1b31b..abbe34e7 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/all_ops_resolver.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/all_ops_resolver.cc @@ -43,6 +43,7 @@ AllOpsResolver::AllOpsResolver() { AddDepthwiseConv2D(); AddDequantize(); AddDetectionPostprocess(); + AddDiv(); AddElu(); AddEqual(); AddEthosU(); @@ -104,6 +105,7 @@ AllOpsResolver::AllOpsResolver() { AddSqueeze(); AddStridedSlice(); AddSub(); + AddSum(); AddSvdf(); AddTanh(); AddTranspose(); diff --git a/code/components/tflite-lib/tensorflow/lite/micro/ibuffer_allocator.h b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/ibuffer_allocator.h similarity index 95% rename from code/components/tflite-lib/tensorflow/lite/micro/ibuffer_allocator.h rename to code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/ibuffer_allocator.h index 3767cb9f..b92d6b2d 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/ibuffer_allocator.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/ibuffer_allocator.h @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_LITE_MICRO_IBUFFER_ALLOCATOR_H_ -#define TENSORFLOW_LITE_MICRO_IBUFFER_ALLOCATOR_H_ +#ifndef TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_IBUFFER_ALLOCATOR_H_ +#define TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_IBUFFER_ALLOCATOR_H_ #include #include @@ -97,4 +97,4 @@ class INonPersistentBufferAllocator { } // namespace tflite -#endif // TENSORFLOW_LITE_MICRO_IBUFFER_ALLOCATOR_H_ +#endif // TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_IBUFFER_ALLOCATOR_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.cc b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.cc new file mode 100644 index 00000000..6389da40 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.cc @@ -0,0 +1,170 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.h" + +#include "tensorflow/lite/micro/memory_helpers.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" + +namespace tflite { + +NonPersistentArenaBufferAllocator::NonPersistentArenaBufferAllocator( + uint8_t* buffer, size_t buffer_size) + : buffer_head_(buffer), + buffer_tail_(buffer + buffer_size), + head_temp_(buffer), + next_temp_(buffer) {} + +NonPersistentArenaBufferAllocator::~NonPersistentArenaBufferAllocator() {} + +// Allocates a temporary buffer. This buffer is not resizable. +uint8_t* NonPersistentArenaBufferAllocator::AllocateTemp(size_t size, + size_t alignment) { + uint8_t* const aligned_result = AlignPointerUp(next_temp_, alignment); + const size_t available_memory = buffer_tail_ - aligned_result; + if (available_memory < size) { + MicroPrintf( + "Failed to allocate temp memory. Requested: %u, " + "available %u, missing: %u", + size, available_memory, size - available_memory); + return nullptr; + } + next_temp_ = aligned_result + size; + temp_buffer_ptr_check_sum_ ^= reinterpret_cast(aligned_result); + temp_buffer_count_++; + return aligned_result; +} + +// Signals that a temporary buffer is no longer needed. +void NonPersistentArenaBufferAllocator::DeallocateTemp(uint8_t* temp_buf) { + temp_buffer_ptr_check_sum_ ^= reinterpret_cast(temp_buf); + temp_buffer_count_--; +} + +// Returns true if all temporary buffers are already deallocated. +bool NonPersistentArenaBufferAllocator::IsAllTempDeallocated() { + if (temp_buffer_count_ != 0 || temp_buffer_ptr_check_sum_ != 0) { + MicroPrintf( + "Number of allocated temp buffers: %d. Checksum passing status: %d", + temp_buffer_count_, !temp_buffer_ptr_check_sum_); + return false; + } + return true; +} + +// Signals that all temporary allocations can be reclaimed. TFLM calls this +// API when it knows that all temporary buffers that it requested has been +// deallocated. The goal of API is to facilitate implementations of +// INonPersistentBufferAllocator can reuse buffer with some reasonable +// complexity. +TfLiteStatus NonPersistentArenaBufferAllocator::ResetTempAllocations() { + if (!IsAllTempDeallocated()) { + MicroPrintf( + "All temp buffers must be freed before calling ResetTempAllocations()"); + return kTfLiteError; + } + next_temp_ = head_temp_; + return kTfLiteOk; +} + +// Returns a buffer that is resizable viable ResizeBuffer(). +uint8_t* NonPersistentArenaBufferAllocator::AllocateResizableBuffer( + size_t size, size_t alignment) { + // Only supports one resizable buffer, which starts at the buffer head. + uint8_t* expected_resizable_buf = AlignPointerUp(buffer_head_, alignment); + + if (resizable_buffer_allocated_) { + MicroPrintf( + "Cannot allocate a new resizable buffer when one is already allocated"); + return nullptr; + } + + if (ResizeBuffer(expected_resizable_buf, size, alignment) == kTfLiteOk) { + resizable_buffer_allocated_ = true; + return expected_resizable_buf; + } + return nullptr; +} + +// Resizes a buffer that is previously returned by the AllocateResizableBuffer. +// Note that ResizeBuffer(old_resizable_buf, 0, 1) effectively deallocates +// a previous allocated resizable buffer. +TfLiteStatus NonPersistentArenaBufferAllocator::ResizeBuffer( + uint8_t* resizable_buf, size_t size, size_t alignment) { + // Only supports one resizable buffer, which starts at the buffer head. + uint8_t* expect_resizable_buf = AlignPointerUp(buffer_head_, alignment); + if (resizable_buf != expect_resizable_buf) { + MicroPrintf("Internal error: buffer is not resizable"); + return kTfLiteError; + } + if (head_temp_ != next_temp_) { + MicroPrintf("ResetTempAllocations() is not called before ResizeBuffer()."); + return kTfLiteError; + } + + const size_t available_memory = buffer_tail_ - expect_resizable_buf; + if (available_memory < size) { + MicroPrintf( + "Failed to resize buffer. Requested: %u, available %u, missing: %u", + size, available_memory, size - available_memory); + return kTfLiteError; + } + head_temp_ = expect_resizable_buf + size; + next_temp_ = head_temp_; + + return kTfLiteOk; +} + +// Frees up the memory occupied by the resizable buffer. +TfLiteStatus NonPersistentArenaBufferAllocator::DeallocateResizableBuffer( + uint8_t* resizable_buf) { + TfLiteStatus status = ResizeBuffer(resizable_buf, 0, 1); + if (status == kTfLiteOk) { + resizable_buffer_allocated_ = false; + } + return status; +} + +// Returns a pointer pointing to the start of the overlay memory, which is +// used for activation tensors and scratch buffers by kernels at Invoke stage. +uint8_t* NonPersistentArenaBufferAllocator::GetOverlayMemoryAddress() const { + return buffer_head_; +} + +// Reserves the size of the overlay memory. This overlay is reserved for the +// kernels at Invoke stage. This is referred to as the overlay because before +// Invoket state, the same memory can be used for temp buffers. The layout of +// the memory is planned by the memory planner separately at Invoke stage. +TfLiteStatus +NonPersistentArenaBufferAllocator::ReserveNonPersistentOverlayMemory( + size_t size, size_t alignment) { + uint8_t* expect_resizable_buf = AlignPointerUp(buffer_head_, alignment); + return ResizeBuffer(expect_resizable_buf, size, alignment); +} + +// Returns the size of non-persistent buffer in use. +size_t NonPersistentArenaBufferAllocator::GetNonPersistentUsedBytes() const { + return (next_temp_ - buffer_head_); +} + +// Returns the number of bytes available with a given alignment. This number +// takes in account any temporary allocations. +size_t NonPersistentArenaBufferAllocator::GetAvailableMemory( + size_t alignment) const { + uint8_t* const aligned_temp = AlignPointerUp(next_temp_, alignment); + uint8_t* const aligned_tail = AlignPointerDown(buffer_tail_, alignment); + return aligned_tail - aligned_temp; +} + +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.h b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.h new file mode 100644 index 00000000..9eb4efeb --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.h @@ -0,0 +1,105 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_NON_PERSISTENT_ARENA_BUFFER_ALLOCATOR_H_ +#define TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_NON_PERSISTENT_ARENA_BUFFER_ALLOCATOR_H_ + +#include +#include + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/micro/arena_allocator/ibuffer_allocator.h" +#include "tensorflow/lite/micro/compatibility.h" + +namespace tflite { + +// Implement INonPersistentBufferAllocator on an arena that is dedicated for +// non-persistent buffers. +class NonPersistentArenaBufferAllocator : public INonPersistentBufferAllocator { + public: + NonPersistentArenaBufferAllocator(uint8_t* buffer, size_t buffer_size); + virtual ~NonPersistentArenaBufferAllocator(); + + // Allocates a temporary buffer. This buffer is not resizable. + uint8_t* AllocateTemp(size_t size, size_t alignment) override; + + // Signals that a temporary buffer is no longer needed. + void DeallocateTemp(uint8_t* buf) override; + + // Returns true if all temporary buffers are already deallocated. + bool IsAllTempDeallocated() override; + + // Signals that all temporary allocations can be reclaimed. TFLM calls this + // API when it knows that all temporary buffers that it requested has been + // deallocated. + TfLiteStatus ResetTempAllocations() override; + + // Returns a buffer that is resizable viable ResizeBuffer(). + uint8_t* AllocateResizableBuffer(size_t size, size_t alignment) override; + + // Resizes a buffer that is previously returned by the + // AllocateResizableBuffer. + TfLiteStatus ResizeBuffer(uint8_t* resizable_buf, size_t size, + size_t alignment) override; + + // Frees up the memory occupied by the resizable buffer. + TfLiteStatus DeallocateResizableBuffer(uint8_t* resizable_buf) override; + + // Returns a pointer pointing to the start of the overlay memory, which is + // used for activation tensors and scratch buffers by kernels at Invoke stage. + uint8_t* GetOverlayMemoryAddress() const override; + + // Reserves the size of the overlay memory. This overlay is reserved for the + // kernels at Invoke stage. This is referred to as the overlay because before + // Invoket state, the same memory can be used for temp buffers. The layout of + // the memory is planned by the memory planner separately at Invoke stage. + TfLiteStatus ReserveNonPersistentOverlayMemory(size_t size, + size_t alignment) override; + + // Returns the size of non-persistent buffer in use. + size_t GetNonPersistentUsedBytes() const override; + + // Returns the number of bytes available with a given alignment. This number + // takes in account any temporary allocations. + size_t GetAvailableMemory(size_t alignment) const override; + + TF_LITE_REMOVE_VIRTUAL_DELETE + + private: + // The memory arena that this allocator manages. + uint8_t* const buffer_head_; + uint8_t* const buffer_tail_; + + // The whole region is split into two parts: + // buffer_head_ to head_temp_ - 1 belongs to the only resizable buffer. + // head_temp_ to buffer_tail_ can be used for (non-resizable) temp buffers. + uint8_t* head_temp_; + + // next_temp_ points to the next available temp buffer allocation address and + // its range is between head_temp_ and buffer_tail_ + uint8_t* next_temp_; + + // XOR Check sum for outstanding temp buffers. + // If all temp buffers are deallocated OR no temp buffers are allocated, + // temp_buffer_ptr_check_sum_ == nullptr. + intptr_t temp_buffer_ptr_check_sum_ = 0; + // Count of outstanding temp buffers. + int temp_buffer_count_ = 0; + bool resizable_buffer_allocated_ = false; +}; + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_NON_PERSISTENT_ARENA_BUFFER_ALLOCATOR_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.cc b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.cc new file mode 100644 index 00000000..0ccc8fb1 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.cc @@ -0,0 +1,52 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.h" + +#include "tensorflow/lite/micro/memory_helpers.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" + +namespace tflite { + +PersistentArenaBufferAllocator::PersistentArenaBufferAllocator( + uint8_t* buffer, size_t buffer_size) + : buffer_head_(buffer), + buffer_tail_(buffer + buffer_size), + tail_temp_(buffer_tail_) {} + +PersistentArenaBufferAllocator::~PersistentArenaBufferAllocator() {} + +uint8_t* PersistentArenaBufferAllocator::AllocatePersistentBuffer( + size_t size, size_t alignment) { + uint8_t* const aligned_result = + AlignPointerDown(tail_temp_ - size, alignment); + if (aligned_result < buffer_head_) { +#ifndef TF_LITE_STRIP_ERROR_STRINGS + const size_t missing_memory = buffer_head_ - aligned_result; + MicroPrintf( + "Failed to allocate tail memory. Requested: %u, " + "available %u, missing: %u", + size, size - missing_memory, missing_memory); +#endif + return nullptr; + } + tail_temp_ = aligned_result; + return aligned_result; +} + +size_t PersistentArenaBufferAllocator::GetPersistentUsedBytes() const { + return buffer_tail_ - tail_temp_; +} + +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.h b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.h new file mode 100644 index 00000000..70de408f --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.h @@ -0,0 +1,59 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_PERSISTENT_ARENA_BUFFER_ALLOCATOR_H_ +#define TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_PERSISTENT_ARENA_BUFFER_ALLOCATOR_H_ + +#include +#include + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/micro/arena_allocator/ibuffer_allocator.h" +#include "tensorflow/lite/micro/compatibility.h" + +namespace tflite { + +// PersistentArenaBufferAllocator is an implementatation of +// IPersistentBufferAllocator interface on an arena that is dedicated for +// persistent buffers. +class PersistentArenaBufferAllocator : public IPersistentBufferAllocator { + public: + PersistentArenaBufferAllocator(uint8_t* buffer, size_t buffer_size); + virtual ~PersistentArenaBufferAllocator(); + + // Allocates persistent memory. The persistent buffer is never freed. + // Returns nullptr if errors occured. + uint8_t* AllocatePersistentBuffer(size_t size, size_t alignment) override; + + // Returns the size of all persistent allocations in bytes. + size_t GetPersistentUsedBytes() const override; + + TF_LITE_REMOVE_VIRTUAL_DELETE + private: + // The memory arena that this allocator manages. + uint8_t* const buffer_head_; + uint8_t* const buffer_tail_; + + // The whole region is split into two parts: + // tail_temp_ to buffer_tail_ contains allocated buffers; + // buffer_head_ to tail_temp_ - 1 belongs to still available spaces. + // So in essence, the allocated region grows from the bottom and emulates + // SingleArenaBufferAllocator's persistent part. + uint8_t* tail_temp_; +}; + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_PERSISTENT_ARENA_BUFFER_ALLOCATOR_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/recording_simple_memory_allocator.cc b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.cc similarity index 55% rename from code/components/tflite-lib/tensorflow/lite/micro/recording_simple_memory_allocator.cc rename to code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.cc index 6d3e72bd..0f24a0b5 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/recording_simple_memory_allocator.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/lite/micro/recording_simple_memory_allocator.h" +#include "tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.h" #include @@ -21,47 +21,49 @@ limitations under the License. namespace tflite { -RecordingSimpleMemoryAllocator::RecordingSimpleMemoryAllocator( +RecordingSingleArenaBufferAllocator::RecordingSingleArenaBufferAllocator( ErrorReporter* error_reporter, uint8_t* buffer_head, size_t buffer_size) - : SimpleMemoryAllocator(error_reporter, buffer_head, buffer_size), + : SingleArenaBufferAllocator(error_reporter, buffer_head, buffer_size), requested_head_bytes_(0), requested_tail_bytes_(0), used_bytes_(0), alloc_count_(0) {} -RecordingSimpleMemoryAllocator::~RecordingSimpleMemoryAllocator() {} +RecordingSingleArenaBufferAllocator::~RecordingSingleArenaBufferAllocator() {} -RecordingSimpleMemoryAllocator* RecordingSimpleMemoryAllocator::Create( - ErrorReporter* error_reporter, uint8_t* buffer_head, size_t buffer_size) { +RecordingSingleArenaBufferAllocator* +RecordingSingleArenaBufferAllocator::Create(ErrorReporter* error_reporter, + uint8_t* buffer_head, + size_t buffer_size) { TFLITE_DCHECK(error_reporter != nullptr); TFLITE_DCHECK(buffer_head != nullptr); - RecordingSimpleMemoryAllocator tmp = - RecordingSimpleMemoryAllocator(error_reporter, buffer_head, buffer_size); + RecordingSingleArenaBufferAllocator tmp = RecordingSingleArenaBufferAllocator( + error_reporter, buffer_head, buffer_size); - uint8_t* allocator_buffer = - tmp.AllocatePersistentBuffer(sizeof(RecordingSimpleMemoryAllocator), - alignof(RecordingSimpleMemoryAllocator)); + uint8_t* allocator_buffer = tmp.AllocatePersistentBuffer( + sizeof(RecordingSingleArenaBufferAllocator), + alignof(RecordingSingleArenaBufferAllocator)); // Use the default copy constructor to populate internal states. - return new (allocator_buffer) RecordingSimpleMemoryAllocator(tmp); + return new (allocator_buffer) RecordingSingleArenaBufferAllocator(tmp); } -size_t RecordingSimpleMemoryAllocator::GetRequestedBytes() const { +size_t RecordingSingleArenaBufferAllocator::GetRequestedBytes() const { return requested_head_bytes_ + requested_tail_bytes_; } -size_t RecordingSimpleMemoryAllocator::GetUsedBytes() const { +size_t RecordingSingleArenaBufferAllocator::GetUsedBytes() const { return used_bytes_; } -size_t RecordingSimpleMemoryAllocator::GetAllocatedCount() const { +size_t RecordingSingleArenaBufferAllocator::GetAllocatedCount() const { return alloc_count_; } -TfLiteStatus RecordingSimpleMemoryAllocator::ResizeBuffer( +TfLiteStatus RecordingSingleArenaBufferAllocator::ResizeBuffer( uint8_t* resizable_buf, size_t size, size_t alignment) { const uint8_t* previous_head = head(); TfLiteStatus status = - SimpleMemoryAllocator::ResizeBuffer(resizable_buf, size, alignment); + SingleArenaBufferAllocator::ResizeBuffer(resizable_buf, size, alignment); if (status == kTfLiteOk) { used_bytes_ += head() - previous_head; requested_head_bytes_ = size; @@ -69,11 +71,11 @@ TfLiteStatus RecordingSimpleMemoryAllocator::ResizeBuffer( return status; } -uint8_t* RecordingSimpleMemoryAllocator::AllocatePersistentBuffer( +uint8_t* RecordingSingleArenaBufferAllocator::AllocatePersistentBuffer( size_t size, size_t alignment) { const uint8_t* previous_tail = tail(); uint8_t* result = - SimpleMemoryAllocator::AllocatePersistentBuffer(size, alignment); + SingleArenaBufferAllocator::AllocatePersistentBuffer(size, alignment); if (result != nullptr) { used_bytes_ += previous_tail - tail(); requested_tail_bytes_ += size; diff --git a/code/components/tflite-lib/tensorflow/lite/micro/recording_simple_memory_allocator.h b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.h similarity index 63% rename from code/components/tflite-lib/tensorflow/lite/micro/recording_simple_memory_allocator.h rename to code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.h index a251e940..3cec561e 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/recording_simple_memory_allocator.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.h @@ -13,28 +13,27 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_LITE_MICRO_RECORDING_SIMPLE_MEMORY_ALLOCATOR_H_ -#define TENSORFLOW_LITE_MICRO_RECORDING_SIMPLE_MEMORY_ALLOCATOR_H_ +#ifndef TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_RECORDING_SINGLE_ARENA_BUFFER_ALLOCATOR_H_ +#define TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_RECORDING_SINGLE_ARENA_BUFFER_ALLOCATOR_H_ +#include "tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h" #include "tensorflow/lite/micro/compatibility.h" -#include "tensorflow/lite/micro/simple_memory_allocator.h" namespace tflite { -// Utility class used to log allocations of a SimpleMemoryAllocator. Should only -// be used in debug/evaluation settings or unit tests to evaluate allocation -// usage. -class RecordingSimpleMemoryAllocator : public SimpleMemoryAllocator { +// Utility class used to log allocations of a SingleArenaBufferAllocator. Should +// only be used in debug/evaluation settings or unit tests to evaluate +// allocation usage. +class RecordingSingleArenaBufferAllocator : public SingleArenaBufferAllocator { public: - RecordingSimpleMemoryAllocator(ErrorReporter* error_reporter, - uint8_t* buffer_head, size_t buffer_size); + RecordingSingleArenaBufferAllocator(ErrorReporter* error_reporter, + uint8_t* buffer_head, size_t buffer_size); // TODO(b/157615197): Cleanup constructors/destructor and use factory // functions. - ~RecordingSimpleMemoryAllocator() override; + ~RecordingSingleArenaBufferAllocator() override; - static RecordingSimpleMemoryAllocator* Create(ErrorReporter* error_reporter, - uint8_t* buffer_head, - size_t buffer_size); + static RecordingSingleArenaBufferAllocator* Create( + ErrorReporter* error_reporter, uint8_t* buffer_head, size_t buffer_size); // Returns the number of bytes requested from the head or tail. size_t GetRequestedBytes() const; @@ -62,4 +61,4 @@ class RecordingSimpleMemoryAllocator : public SimpleMemoryAllocator { } // namespace tflite -#endif // TENSORFLOW_LITE_MICRO_RECORDING_SIMPLE_MEMORY_ALLOCATOR_H_ +#endif // TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_RECORDING_SINGLE_ARENA_BUFFER_ALLOCATOR_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/simple_memory_allocator.cc b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.cc similarity index 66% rename from code/components/tflite-lib/tensorflow/lite/micro/simple_memory_allocator.cc rename to code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.cc index e5d87afb..15d512bd 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/simple_memory_allocator.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/lite/micro/simple_memory_allocator.h" +#include "tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h" #include #include @@ -29,42 +29,45 @@ limitations under the License. namespace tflite { -SimpleMemoryAllocator::SimpleMemoryAllocator(ErrorReporter* error_reporter, - uint8_t* buffer_head, - uint8_t* buffer_tail) - : error_reporter_(error_reporter), +SingleArenaBufferAllocator::SingleArenaBufferAllocator( + ErrorReporter* error_reporter, uint8_t* buffer_head, uint8_t* buffer_tail) + : +#if !defined(TF_LITE_STRIP_ERROR_STRINGS) + error_reporter_(error_reporter), +#endif buffer_head_(buffer_head), buffer_tail_(buffer_tail), head_(buffer_head), tail_(buffer_tail), - temp_(buffer_head_) {} + temp_(buffer_head_) { +} -SimpleMemoryAllocator::SimpleMemoryAllocator(ErrorReporter* error_reporter, - uint8_t* buffer, - size_t buffer_size) - : SimpleMemoryAllocator(error_reporter, buffer, buffer + buffer_size) {} +SingleArenaBufferAllocator::SingleArenaBufferAllocator( + ErrorReporter* error_reporter, uint8_t* buffer, size_t buffer_size) + : SingleArenaBufferAllocator(error_reporter, buffer, buffer + buffer_size) { +} /* static */ -SimpleMemoryAllocator* SimpleMemoryAllocator::Create( +SingleArenaBufferAllocator* SingleArenaBufferAllocator::Create( ErrorReporter* error_reporter, uint8_t* buffer_head, size_t buffer_size) { TFLITE_DCHECK(error_reporter != nullptr); TFLITE_DCHECK(buffer_head != nullptr); - SimpleMemoryAllocator tmp = - SimpleMemoryAllocator(error_reporter, buffer_head, buffer_size); + SingleArenaBufferAllocator tmp = + SingleArenaBufferAllocator(error_reporter, buffer_head, buffer_size); - // Allocate enough bytes from the buffer to create a SimpleMemoryAllocator. - // The new instance will use the current adjusted tail buffer from the tmp - // allocator instance. + // Allocate enough bytes from the buffer to create a + // SingleArenaBufferAllocator. The new instance will use the current adjusted + // tail buffer from the tmp allocator instance. uint8_t* allocator_buffer = tmp.AllocatePersistentBuffer( - sizeof(SimpleMemoryAllocator), alignof(SimpleMemoryAllocator)); + sizeof(SingleArenaBufferAllocator), alignof(SingleArenaBufferAllocator)); // Use the default copy constructor to populate internal states. - return new (allocator_buffer) SimpleMemoryAllocator(tmp); + return new (allocator_buffer) SingleArenaBufferAllocator(tmp); } -SimpleMemoryAllocator::~SimpleMemoryAllocator() {} +SingleArenaBufferAllocator::~SingleArenaBufferAllocator() {} -uint8_t* SimpleMemoryAllocator::AllocateResizableBuffer(size_t size, - size_t alignment) { +uint8_t* SingleArenaBufferAllocator::AllocateResizableBuffer(size_t size, + size_t alignment) { // Only supports one resizable buffer, which starts at the buffer head. uint8_t* expect_resizable_buf = AlignPointerUp(buffer_head_, alignment); if (ResizeBuffer(expect_resizable_buf, size, alignment) == kTfLiteOk) { @@ -73,20 +76,20 @@ uint8_t* SimpleMemoryAllocator::AllocateResizableBuffer(size_t size, return nullptr; } -TfLiteStatus SimpleMemoryAllocator::DeallocateResizableBuffer( +TfLiteStatus SingleArenaBufferAllocator::DeallocateResizableBuffer( uint8_t* resizable_buf) { return ResizeBuffer(resizable_buf, 0, 1); } -TfLiteStatus SimpleMemoryAllocator::ReserveNonPersistentOverlayMemory( +TfLiteStatus SingleArenaBufferAllocator::ReserveNonPersistentOverlayMemory( size_t size, size_t alignment) { uint8_t* expect_resizable_buf = AlignPointerUp(buffer_head_, alignment); return ResizeBuffer(expect_resizable_buf, size, alignment); } -TfLiteStatus SimpleMemoryAllocator::ResizeBuffer(uint8_t* resizable_buf, - size_t size, - size_t alignment) { +TfLiteStatus SingleArenaBufferAllocator::ResizeBuffer(uint8_t* resizable_buf, + size_t size, + size_t alignment) { // Only supports one resizable buffer, which starts at the buffer head. uint8_t* expect_resizable_buf = AlignPointerUp(buffer_head_, alignment); if (head_ != temp_ || resizable_buf != expect_resizable_buf) { @@ -112,8 +115,8 @@ TfLiteStatus SimpleMemoryAllocator::ResizeBuffer(uint8_t* resizable_buf, return kTfLiteOk; } -uint8_t* SimpleMemoryAllocator::AllocatePersistentBuffer(size_t size, - size_t alignment) { +uint8_t* SingleArenaBufferAllocator::AllocatePersistentBuffer( + size_t size, size_t alignment) { uint8_t* const aligned_result = AlignPointerDown(tail_ - size, alignment); if (aligned_result < head_) { #ifndef TF_LITE_STRIP_ERROR_STRINGS @@ -129,7 +132,8 @@ uint8_t* SimpleMemoryAllocator::AllocatePersistentBuffer(size_t size, return aligned_result; } -uint8_t* SimpleMemoryAllocator::AllocateTemp(size_t size, size_t alignment) { +uint8_t* SingleArenaBufferAllocator::AllocateTemp(size_t size, + size_t alignment) { uint8_t* const aligned_result = AlignPointerUp(temp_, alignment); const size_t available_memory = tail_ - aligned_result; if (available_memory < size) { @@ -145,12 +149,12 @@ uint8_t* SimpleMemoryAllocator::AllocateTemp(size_t size, size_t alignment) { return aligned_result; } -void SimpleMemoryAllocator::DeallocateTemp(uint8_t* temp_buf) { +void SingleArenaBufferAllocator::DeallocateTemp(uint8_t* temp_buf) { temp_buffer_ptr_check_sum_ ^= (reinterpret_cast(temp_buf)); temp_buffer_count_--; } -bool SimpleMemoryAllocator::IsAllTempDeallocated() { +bool SingleArenaBufferAllocator::IsAllTempDeallocated() { if (temp_buffer_count_ != 0 || temp_buffer_ptr_check_sum_ != 0) { MicroPrintf( "Number of allocated temp buffers: %d. Checksum passing status: %d", @@ -160,7 +164,7 @@ bool SimpleMemoryAllocator::IsAllTempDeallocated() { return true; } -TfLiteStatus SimpleMemoryAllocator::ResetTempAllocations() { +TfLiteStatus SingleArenaBufferAllocator::ResetTempAllocations() { // TODO(b/209453859): enable error check based on IsAllTempDeallocated after // all AllocateTemp have been paird with DeallocateTemp if (!IsAllTempDeallocated()) { @@ -172,34 +176,34 @@ TfLiteStatus SimpleMemoryAllocator::ResetTempAllocations() { return kTfLiteOk; } -uint8_t* SimpleMemoryAllocator::GetOverlayMemoryAddress() const { +uint8_t* SingleArenaBufferAllocator::GetOverlayMemoryAddress() const { return buffer_head_; } -size_t SimpleMemoryAllocator::GetNonPersistentUsedBytes() const { +size_t SingleArenaBufferAllocator::GetNonPersistentUsedBytes() const { return std::max(head_ - buffer_head_, temp_ - buffer_head_); } -size_t SimpleMemoryAllocator::GetPersistentUsedBytes() const { +size_t SingleArenaBufferAllocator::GetPersistentUsedBytes() const { return buffer_tail_ - tail_; } -size_t SimpleMemoryAllocator::GetAvailableMemory(size_t alignment) const { +size_t SingleArenaBufferAllocator::GetAvailableMemory(size_t alignment) const { uint8_t* const aligned_temp = AlignPointerUp(temp_, alignment); uint8_t* const aligned_tail = AlignPointerDown(tail_, alignment); return aligned_tail - aligned_temp; } -size_t SimpleMemoryAllocator::GetUsedBytes() const { +size_t SingleArenaBufferAllocator::GetUsedBytes() const { return GetPersistentUsedBytes() + GetNonPersistentUsedBytes(); } -size_t SimpleMemoryAllocator::GetBufferSize() const { +size_t SingleArenaBufferAllocator::GetBufferSize() const { return buffer_tail_ - buffer_head_; } -uint8_t* SimpleMemoryAllocator::head() const { return head_; } +uint8_t* SingleArenaBufferAllocator::head() const { return head_; } -uint8_t* SimpleMemoryAllocator::tail() const { return tail_; } +uint8_t* SingleArenaBufferAllocator::tail() const { return tail_; } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/simple_memory_allocator.h b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h similarity index 83% rename from code/components/tflite-lib/tensorflow/lite/micro/simple_memory_allocator.h rename to code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h index d88c4a3d..d3be1f23 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/simple_memory_allocator.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h @@ -13,37 +13,37 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_LITE_MICRO_SIMPLE_MEMORY_ALLOCATOR_H_ -#define TENSORFLOW_LITE_MICRO_SIMPLE_MEMORY_ALLOCATOR_H_ +#ifndef TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_SINGLE_ARENA_BUFFER_ALLOCATOR_H_ +#define TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_SINGLE_ARENA_BUFFER_ALLOCATOR_H_ #include #include #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/micro/arena_allocator/ibuffer_allocator.h" #include "tensorflow/lite/micro/compatibility.h" -#include "tensorflow/lite/micro/ibuffer_allocator.h" namespace tflite { // TODO(petewarden): This allocator never frees up or reuses any memory, even // though we have enough information about lifetimes of the tensors to do so. // This makes it pretty wasteful, so we should use a more intelligent method. -class SimpleMemoryAllocator : public INonPersistentBufferAllocator, - public IPersistentBufferAllocator { +class SingleArenaBufferAllocator : public INonPersistentBufferAllocator, + public IPersistentBufferAllocator { public: // TODO(b/157615197): Cleanup constructors/destructor and use factory // functions. - SimpleMemoryAllocator(ErrorReporter* error_reporter, uint8_t* buffer_head, - uint8_t* buffer_tail); - SimpleMemoryAllocator(ErrorReporter* error_reporter, uint8_t* buffer, - size_t buffer_size); - virtual ~SimpleMemoryAllocator(); + SingleArenaBufferAllocator(ErrorReporter* error_reporter, + uint8_t* buffer_head, uint8_t* buffer_tail); + SingleArenaBufferAllocator(ErrorReporter* error_reporter, uint8_t* buffer, + size_t buffer_size); + virtual ~SingleArenaBufferAllocator(); - // Creates a new SimpleMemoryAllocator from a given buffer head and size. - static SimpleMemoryAllocator* Create(ErrorReporter* error_reporter, - uint8_t* buffer_head, - size_t buffer_size); + // Creates a new SingleArenaBufferAllocator from a given buffer head and size. + static SingleArenaBufferAllocator* Create(ErrorReporter* error_reporter, + uint8_t* buffer_head, + size_t buffer_size); // Resizes a buffer that is previously returned by the // AllocateResizableBuffer. In current implementation, it Adjusts the head @@ -126,7 +126,9 @@ class SimpleMemoryAllocator : public INonPersistentBufferAllocator, private: size_t GetBufferSize() const; +#if !defined(TF_LITE_STRIP_ERROR_STRINGS) ErrorReporter* error_reporter_; +#endif uint8_t* buffer_head_; uint8_t* buffer_tail_; uint8_t* head_; @@ -147,4 +149,4 @@ class SimpleMemoryAllocator : public INonPersistentBufferAllocator, } // namespace tflite -#endif // TENSORFLOW_LITE_MICRO_SIMPLE_MEMORY_ALLOCATOR_H_ +#endif // TENSORFLOW_LITE_MICRO_ARENA_ALLOCATOR_SINGLE_ARENA_BUFFER_ALLOCATOR_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/fake_micro_context.cc b/code/components/tflite-lib/tensorflow/lite/micro/fake_micro_context.cc index 5a5ba9ab..2403c6b1 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/fake_micro_context.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/fake_micro_context.cc @@ -16,10 +16,10 @@ limitations under the License. #include "tensorflow/lite/micro/fake_micro_context.h" #include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h" #include "tensorflow/lite/micro/micro_allocator.h" #include "tensorflow/lite/micro/micro_arena_constants.h" #include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/simple_memory_allocator.h" namespace tflite { namespace { @@ -30,7 +30,7 @@ static uint8_t dummy_tensor_arena[KDummyTensorArenaSize]; } // namespace FakeMicroContext::FakeMicroContext(TfLiteTensor* tensors, - SimpleMemoryAllocator* allocator, + SingleArenaBufferAllocator* allocator, MicroGraph* micro_graph) : MicroContext( MicroAllocator::Create(dummy_tensor_arena, KDummyTensorArenaSize, @@ -67,10 +67,10 @@ TfLiteEvalTensor* FakeMicroContext::GetEvalTensor(int tensor_index) { } void* FakeMicroContext::AllocatePersistentBuffer(size_t bytes) { - // FakeMicroContext use SimpleMemoryAllocator, which does not automatically - // apply the buffer alignment like MicroAllocator. - // The buffer alignment is potentially wasteful but allows the - // fake_micro_context to work correctly with optimized kernels. + // FakeMicroContext use SingleArenaBufferAllocator, which does not + // automatically apply the buffer alignment like MicroAllocator. The buffer + // alignment is potentially wasteful but allows the fake_micro_context to work + // correctly with optimized kernels. return allocator_->AllocatePersistentBuffer(bytes, MicroArenaBufferAlignment()); } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/fake_micro_context.h b/code/components/tflite-lib/tensorflow/lite/micro/fake_micro_context.h index 99933c19..31b39d38 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/fake_micro_context.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/fake_micro_context.h @@ -23,7 +23,7 @@ namespace tflite { // A fake of MicroContext for kernel util tests. class FakeMicroContext : public MicroContext { public: - FakeMicroContext(TfLiteTensor* tensors, SimpleMemoryAllocator* allocator, + FakeMicroContext(TfLiteTensor* tensors, SingleArenaBufferAllocator* allocator, MicroGraph* micro_graph); void* AllocatePersistentBuffer(size_t bytes) override; @@ -46,7 +46,7 @@ class FakeMicroContext : public MicroContext { TfLiteTensor* tensors_; int allocated_tensor_count_ = 0; - SimpleMemoryAllocator* allocator_; + SingleArenaBufferAllocator* allocator_; TF_LITE_REMOVE_VIRTUAL_DELETE }; diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/activations.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/activations.cc index c556ac64..e0b79631 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/activations.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/activations.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" #include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" #include "tensorflow/lite/micro/micro_utils.h" namespace tflite { @@ -60,8 +61,8 @@ TfLiteStatus ReluEval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } default: { - TF_LITE_KERNEL_LOG(context, "Only float32 is supported currently, got %s", - TfLiteTypeGetName(input->type)); + MicroPrintf("Only float32 is supported currently, got %s", + TfLiteTypeGetName(input->type)); return kTfLiteError; } } @@ -99,8 +100,8 @@ TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } default: { - TF_LITE_KERNEL_LOG(context, "Only float32 is supported currently, got %s", - TfLiteTypeGetName(input->type)); + MicroPrintf("Only float32 is supported currently, got %s", + TfLiteTypeGetName(input->type)); return kTfLiteError; } } @@ -109,25 +110,11 @@ TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_RELU() { - return {/*init=*/ReluInit, - /*free=*/nullptr, - /*prepare=*/ReluPrepare, - /*invoke=*/ReluEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(ReluInit, ReluPrepare, ReluEval); } TfLiteRegistration Register_RELU6() { - return {/*init=*/Relu6Init, - /*free=*/nullptr, - /*prepare=*/Relu6Prepare, - /*invoke=*/Relu6Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Relu6Init, Relu6Prepare, Relu6Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/add.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/add.cc index 75523d14..f75db4e5 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/add.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/add.cc @@ -159,14 +159,7 @@ TfLiteStatus AddEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration Register_ADD() { - return {/*init=*/AddInit, - /*free=*/nullptr, - /*prepare=*/AddPrepare, - /*invoke=*/AddEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(AddInit, AddPrepare, AddEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/add_n.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/add_n.cc index 5d0ab724..ce064687 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/add_n.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/add_n.cc @@ -208,14 +208,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_ADD_N() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/arg_min_max.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/arg_min_max.cc index 8217a4a0..a8aa5a48 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/arg_min_max.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/arg_min_max.cc @@ -104,25 +104,11 @@ TfLiteStatus ArgMaxEval(TfLiteContext* context, TfLiteNode* node) { } // namespace arg_min_max TfLiteRegistration Register_ARG_MAX() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/arg_min_max::ArgMaxEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, nullptr, arg_min_max::ArgMaxEval); } TfLiteRegistration Register_ARG_MIN() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/arg_min_max::ArgMinEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, nullptr, arg_min_max::ArgMinEval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/assign_variable.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/assign_variable.cc index e28ebebb..a770d0aa 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/assign_variable.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/assign_variable.cc @@ -95,14 +95,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace. TfLiteRegistration Register_ASSIGN_VARIABLE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/batch_to_space_nd.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/batch_to_space_nd.cc index 07b680df..be82d942 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/batch_to_space_nd.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/batch_to_space_nd.cc @@ -105,14 +105,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace. TfLiteRegistration Register_BATCH_TO_SPACE_ND() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/broadcast_args.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/broadcast_args.cc index fa333249..be2672ec 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/broadcast_args.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/broadcast_args.cc @@ -84,14 +84,8 @@ TfLiteStatus BroadcastArgsEval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_BROADCAST_ARGS() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/BroadcastArgsPrepare, - /*invoke=*/BroadcastArgsEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, BroadcastArgsPrepare, + BroadcastArgsEval); } -} // namespace tflite \ No newline at end of file +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/broadcast_to.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/broadcast_to.cc index 5302faf1..63a14db2 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/broadcast_to.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/broadcast_to.cc @@ -116,14 +116,8 @@ TfLiteStatus BroadcastToEval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_BROADCAST_TO() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/BroadcastToPrepare, - /*invoke=*/BroadcastToEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, BroadcastToPrepare, + BroadcastToEval); } -} // namespace tflite \ No newline at end of file +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/call_once.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/call_once.cc index 4db39f7d..200242b2 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/call_once.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/call_once.cc @@ -82,14 +82,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace. TfLiteRegistration Register_CALL_ONCE() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/cast.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/cast.cc index dc651a24..a1f4516b 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/cast.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/cast.cc @@ -108,14 +108,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_CAST() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/ceil.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/ceil.cc index d0a48f91..a390a735 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/ceil.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/ceil.cc @@ -67,14 +67,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace ceil TfLiteRegistration Register_CEIL() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ceil::Prepare, - /*invoke=*/ceil::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, ceil::Prepare, ceil::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/circular_buffer.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/circular_buffer.cc index bda3e66a..399d1648 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/circular_buffer.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/circular_buffer.cc @@ -108,14 +108,8 @@ TfLiteStatus CircularBufferEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration* Register_CIRCULAR_BUFFER() { - static TfLiteRegistration r = {/*init=*/CircularBufferInit, - /*free=*/nullptr, - /*prepare=*/CircularBufferPrepare, - /*invoke=*/CircularBufferEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + static TfLiteRegistration r = tflite::micro::RegisterOp( + CircularBufferInit, CircularBufferPrepare, CircularBufferEval); return &r; } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/circular_buffer_common.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/circular_buffer_common.cc index 682efb43..81db6e65 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/circular_buffer_common.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/circular_buffer_common.cc @@ -39,13 +39,12 @@ const int kCircularBufferCyclesMaxIndex = 0; // 'cycles_max' const TfLiteStatus kTfLiteAbort = static_cast(-9); TfLiteStatus CircularBufferPrepare(TfLiteContext* context, TfLiteNode* node) { + MicroContext* micro_context = GetMicroContext(context); - MicroContext * micro_context = GetMicroContext(context); - - TfLiteTensor* input = - micro_context-> AllocateTempInputTensor(node, kCircularBufferInputTensor); - TfLiteTensor* output = - micro_context-> AllocateTempOutputTensor(node, kCircularBufferOutputTensor); + TfLiteTensor* input = + micro_context->AllocateTempInputTensor(node, kCircularBufferInputTensor); + TfLiteTensor* output = micro_context->AllocateTempOutputTensor( + node, kCircularBufferOutputTensor); TFLITE_DCHECK(node->user_data != nullptr); OpDataCircularBuffer* op_data = diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/comparisons.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/comparisons.cc index 925c3fb5..cff15e4d 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/comparisons.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/comparisons.cc @@ -583,69 +583,33 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } // namespace comparisons TfLiteRegistration Register_EQUAL() { - return {/*init=*/comparisons::Init, - /*free=*/nullptr, - /*prepare=*/comparisons::Prepare, - /*invoke=*/comparisons::EqualEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(comparisons::Init, comparisons::Prepare, + comparisons::EqualEval); } TfLiteRegistration Register_NOT_EQUAL() { - return {/*init=*/comparisons::Init, - /*free=*/nullptr, - /*prepare=*/comparisons::Prepare, - /*invoke=*/comparisons::NotEqualEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(comparisons::Init, comparisons::Prepare, + comparisons::NotEqualEval); } TfLiteRegistration Register_GREATER() { - return {/*init=*/comparisons::Init, - /*free=*/nullptr, - /*prepare=*/comparisons::Prepare, - /*invoke=*/comparisons::GreaterEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(comparisons::Init, comparisons::Prepare, + comparisons::GreaterEval); } TfLiteRegistration Register_GREATER_EQUAL() { - return {/*init=*/comparisons::Init, - /*free=*/nullptr, - /*prepare=*/comparisons::Prepare, - /*invoke=*/comparisons::GreaterEqualEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(comparisons::Init, comparisons::Prepare, + comparisons::GreaterEqualEval); } TfLiteRegistration Register_LESS() { - return {/*init=*/comparisons::Init, - /*free=*/nullptr, - /*prepare=*/comparisons::Prepare, - /*invoke=*/comparisons::LessEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(comparisons::Init, comparisons::Prepare, + comparisons::LessEval); } TfLiteRegistration Register_LESS_EQUAL() { - return {/*init=*/comparisons::Init, - /*free=*/nullptr, - /*prepare=*/comparisons::Prepare, - /*invoke=*/comparisons::LessEqualEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(comparisons::Init, comparisons::Prepare, + comparisons::LessEqualEval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/concatenation.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/concatenation.cc index d727a0d5..34622c22 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/concatenation.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/concatenation.cc @@ -148,12 +148,12 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE(context, input != nullptr); int num_dimensions = NumDimensions(input); - if (num_dimensions > 4) { + if (num_dimensions > RuntimeShape::kMaxSmallSize) { TF_LITE_KERNEL_LOG( context, - "Op Concatenation does not currently support num dimensions >4 " + "Op Concatenation does not currently support num dimensions > %d " "Tensor has %d dimensions.", - num_dimensions); + RuntimeShape::kMaxSmallSize, num_dimensions); return kTfLiteError; } micro_context->DeallocateTempTfLiteTensor(input); @@ -252,14 +252,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace concatenation TfLiteRegistration Register_CONCATENATION() { - return {/*init=*/concatenation::Init, - /*free=*/nullptr, - /*prepare=*/concatenation::Prepare, - /*invoke=*/concatenation::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(concatenation::Init, concatenation::Prepare, + concatenation::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/conv.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/conv.cc index 0fed1223..87ea92e6 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/conv.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/conv.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/padding.h" #include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" namespace tflite { namespace { @@ -67,23 +68,47 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorShape(filter), tflite::micro::GetTensorData(filter), tflite::micro::GetTensorShape(bias), - tflite::micro::GetTensorData(bias), + tflite::micro::GetOptionalTensorData(bias), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output), tflite::micro::GetTensorShape(nullptr), nullptr); break; } case kTfLiteInt16: { - reference_integer_ops::ConvPerChannel( - ConvParamsQuantized(params, data), data.per_channel_output_multiplier, - data.per_channel_output_shift, tflite::micro::GetTensorShape(input), - tflite::micro::GetTensorData(input), - tflite::micro::GetTensorShape(filter), - tflite::micro::GetTensorData(filter), - tflite::micro::GetTensorShape(bias), - tflite::micro::GetTensorData(bias), - tflite::micro::GetTensorShape(output), - tflite::micro::GetTensorData(output)); + switch (bias->type) { + case kTfLiteInt32: { + reference_integer_ops::ConvPerChannel( + ConvParamsQuantized(params, data), + data.per_channel_output_multiplier, data.per_channel_output_shift, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetOptionalTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + break; + } + case kTfLiteInt64: { + reference_integer_ops::ConvPerChannel( + ConvParamsQuantized(params, data), + data.per_channel_output_multiplier, data.per_channel_output_shift, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), + tflite::micro::GetOptionalTensorData(bias), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + break; + } + default: + MicroPrintf("Bias type %s (%d) not supported.", + TfLiteTypeGetName(bias->type), bias->type); + return kTfLiteError; + } break; } case kTfLiteInt8: { @@ -94,14 +119,14 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorShape(filter), tflite::micro::GetTensorData(filter), tflite::micro::GetTensorShape(bias), - tflite::micro::GetTensorData(bias), + tflite::micro::GetOptionalTensorData(bias), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); break; } default: - TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", - TfLiteTypeGetName(input->type), input->type); + MicroPrintf("Type %s (%d) not supported.", TfLiteTypeGetName(input->type), + input->type); return kTfLiteError; } return kTfLiteOk; @@ -110,14 +135,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_CONV_2D() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/ConvPrepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, ConvPrepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/conv_test.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/conv_test.h index 38b69525..47ba8ac4 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/conv_test.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/conv_test.h @@ -97,6 +97,16 @@ TfLiteStatus TestConvQuantizedPerChannel( float output_scale, int output_zero_point, TfLiteConvParams* conv_params, TfLiteRegistration registration, int16_t* output_data); +TfLiteStatus TestConvQuantizedPerChannel( + int* input_dims_data, const float* input_data, int16_t* input_quantized, + float input_scale, int input_zero_point, int* filter_dims_data, + const float* filter_data, int8_t* filter_data_quantized, + int* bias_dims_data, const float* bias_data, int32_t* bias_data_quantized, + float* bias_scales, int* bias_zero_points, int* output_dims_data, + const float* expected_output_data, int16_t* expected_output_data_quantized, + float output_scale, int output_zero_point, TfLiteConvParams* conv_params, + TfLiteRegistration registration, int16_t* output_data); + } // namespace testing } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/cumsum.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/cumsum.cc index 61f7af23..eedc61fd 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/cumsum.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/cumsum.cc @@ -169,14 +169,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_CUMSUM() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/depth_to_space.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/depth_to_space.cc index cce93c9c..ec000540 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/depth_to_space.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/depth_to_space.cc @@ -136,14 +136,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_DEPTH_TO_SPACE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/depthwise_conv.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/depthwise_conv.cc index 8a58433a..d2468ff9 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/depthwise_conv.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/depthwise_conv.cc @@ -62,7 +62,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorShape(filter), tflite::micro::GetTensorData(filter), tflite::micro::GetTensorShape(bias), - tflite::micro::GetTensorData(bias), + tflite::micro::GetOptionalTensorData(bias), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); break; @@ -76,7 +76,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorShape(filter), tflite::micro::GetTensorData(filter), tflite::micro::GetTensorShape(bias), - tflite::micro::GetTensorData(bias), + tflite::micro::GetOptionalTensorData(bias), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); break; @@ -92,14 +92,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_DEPTHWISE_CONV_2D() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/DepthwiseConvPrepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, DepthwiseConvPrepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/depthwise_conv.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/depthwise_conv.h index 7a7eb0ba..562438d7 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/depthwise_conv.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/depthwise_conv.h @@ -1,4 +1,4 @@ -/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -49,6 +49,32 @@ TfLiteStatus CalculateOpDataDepthwiseConv( TfLiteStatus DepthwiseConvPrepare(TfLiteContext* context, TfLiteNode* node); +// This is the most generic TfLiteRegistration. The actual supported types may +// still be target dependent. The only requirement is that every implementation +// (reference or optimized) must define this function. +TfLiteRegistration Register_DEPTHWISE_CONV_2D(); + +#if defined(CMSIS_NN) +// Returns a TfLiteRegistration struct for kernel variant that only supports +// int8 activations and int8 weights and uses the latency optimized +// implementations. +TfLiteRegistration Register_DEPTHWISE_CONV_2D_INT8(); + +// Returns a TfLiteRegistration struct for kernel variant that only supports +// int16 activations and int8 weights and uses the latency optimized +// implementations. +TfLiteRegistration Register_DEPTHWISE_CONV_2D_INT16(); + +#else +inline TfLiteRegistration Register_DEPTHWISE_CONV_2D_INT8() { + return Register_DEPTHWISE_CONV_2D(); +} + +inline TfLiteRegistration Register_DEPTHWISE_CONV_2D_INT16() { + return Register_DEPTHWISE_CONV_2D(); +} +#endif + } // namespace tflite #endif // TENSORFLOW_LITE_MICRO_KERNELS_DEPTHWISE_CONV_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/dequantize.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/dequantize.cc index 4438ea33..1cf7f133 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/dequantize.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/dequantize.cc @@ -57,6 +57,13 @@ TfLiteStatus DequantizeEval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); break; + case kTfLiteUInt8: + reference_ops::Dequantize(data->quantization_params, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + break; default: MicroPrintf("Input %s, output %s not supported.", TfLiteTypeGetName(input->type), @@ -74,14 +81,8 @@ TfLiteStatus DequantizeEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration Register_DEQUANTIZE() { - return {/*init=*/DequantizeInit, - /*free=*/nullptr, - /*prepare=*/DequantizePrepare, - /*invoke=*/DequantizeEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(DequantizeInit, DequantizePrepare, + DequantizeEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/dequantize_common.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/dequantize_common.cc index 4be5ad89..438f9cda 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/dequantize_common.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/dequantize_common.cc @@ -41,8 +41,9 @@ TfLiteStatus DequantizePrepare(TfLiteContext* context, TfLiteNode* node) { TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); TF_LITE_ENSURE(context, output != nullptr); - TF_LITE_ENSURE(context, - input->type == kTfLiteInt8 || input->type == kTfLiteInt16); + TF_LITE_ENSURE(context, input->type == kTfLiteInt8 || + input->type == kTfLiteInt16 || + input->type == kTfLiteUInt8); TF_LITE_ENSURE(context, output->type == kTfLiteFloat32); if (output->type == kTfLiteInt32) { diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/detection_postprocess.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/detection_postprocess.cc index efe57e2f..326d87b5 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/detection_postprocess.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/detection_postprocess.cc @@ -149,8 +149,6 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) { return op_data; } -void Free(TfLiteContext* context, void* buffer) {} - TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { auto* op_data = static_cast(node->user_data); @@ -802,14 +800,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration* Register_DETECTION_POSTPROCESS() { - static TfLiteRegistration r = {/*init=*/Init, - /*free=*/Free, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + static TfLiteRegistration r = tflite::micro::RegisterOp(Init, Prepare, Eval); return &r; } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/div.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/div.cc new file mode 100644 index 00000000..099c0225 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/div.cc @@ -0,0 +1,208 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/kernels/internal/reference/div.h" + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h" +#include "tensorflow/lite/kernels/internal/types.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" + +namespace tflite { +namespace { + +constexpr int kInputTensor1 = 0; +constexpr int kInputTensor2 = 1; +constexpr int kOutputTensor = 0; + +struct OpDataDiv { + // Parameters used in the quantized paths where the output is 8bit + int32_t input1_zero_point; + int32_t input2_zero_point; + int32_t output_zero_point; + int32_t output_activation_min; + int32_t output_activation_max; + + // Parameters used in all quantized paths + int32_t output_multiplier; + int output_shift; +}; + +TfLiteStatus CalculateOpDataDiv(TfLiteContext* context, TfLiteTensor* input1, + TfLiteTensor* input2, TfLiteTensor* output, + TfLiteDivParams* params, OpDataDiv* data) { + TF_LITE_ENSURE_TYPES_EQ(context, input1->type, input2->type); + TF_LITE_ENSURE_TYPES_EQ(context, input1->type, output->type); + + if (output->type == kTfLiteInt8) { + TF_LITE_ENSURE_STATUS(CalculateActivationRangeQuantized( + context, params->activation, output, &data->output_activation_min, + &data->output_activation_max)); + const double real_multiplier = static_cast( + input1->params.scale / (input2->params.scale * output->params.scale)); + QuantizeMultiplier(real_multiplier, &data->output_multiplier, + &data->output_shift); + data->input1_zero_point = input1->params.zero_point; + data->input2_zero_point = input2->params.zero_point; + data->output_zero_point = output->params.zero_point; + } + + return kTfLiteOk; +} + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpDataDiv)); +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); + + MicroContext* micro_context = GetMicroContext(context); + TfLiteTensor* input1 = + micro_context->AllocateTempInputTensor(node, kInputTensor1); + TF_LITE_ENSURE(context, input1 != nullptr); + TfLiteTensor* input2 = + micro_context->AllocateTempInputTensor(node, kInputTensor2); + TF_LITE_ENSURE(context, input2 != nullptr); + TfLiteTensor* output = + micro_context->AllocateTempOutputTensor(node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + OpDataDiv* data = static_cast(node->user_data); + auto* params = reinterpret_cast(node->builtin_data); + + TF_LITE_ENSURE_STATUS( + CalculateOpDataDiv(context, input1, input2, output, params, data)); + + micro_context->DeallocateTempTfLiteTensor(input1); + micro_context->DeallocateTempTfLiteTensor(input2); + micro_context->DeallocateTempTfLiteTensor(output); + return kTfLiteOk; +} + +void EvalDiv(TfLiteContext* context, TfLiteNode* node, TfLiteDivParams* params, + const OpDataDiv* data, const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, TfLiteEvalTensor* output) { + tflite::ArithmeticParams op_params = {}; + +#define TF_LITE_DIV(type, opname, data_type) \ + data_type output_activation_min, output_activation_max; \ + CalculateActivationRange(params->activation, &output_activation_min, \ + &output_activation_max); \ + SetActivationParams(output_activation_min, output_activation_max, \ + &op_params); \ + type::opname(op_params, tflite::micro::GetTensorShape(input1), \ + tflite::micro::GetTensorData(input1), \ + tflite::micro::GetTensorShape(input2), \ + tflite::micro::GetTensorData(input2), \ + tflite::micro::GetTensorShape(output), \ + tflite::micro::GetTensorData(output)) + + bool requires_broadcast = reference_ops::ProcessBroadcastShapes( + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorShape(input2), &op_params); + + if (requires_broadcast) { + TF_LITE_DIV(reference_ops, BroadcastDivSlow, float); + } else { + TF_LITE_DIV(reference_ops, Div, float); + } +#undef TF_LITE_DIV +} + +TfLiteStatus EvalQuantized(TfLiteContext* context, TfLiteNode* node, + TfLiteDivParams* params, const OpDataDiv* data, + const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, + TfLiteEvalTensor* output) { + tflite::ArithmeticParams op_params = {}; + +#define TF_LITE_DIV(type, opname, dtype) \ + type::opname(op_params, tflite::micro::GetTensorShape(input1), \ + tflite::micro::GetTensorData(input1), \ + tflite::micro::GetTensorShape(input2), \ + tflite::micro::GetTensorData(input2), \ + tflite::micro::GetTensorShape(output), \ + tflite::micro::GetTensorData(output)) + + if (input1->type == kTfLiteInt8 && input2->type == kTfLiteInt8 && + output->type == kTfLiteInt8) { + SetActivationParams(data->output_activation_min, + data->output_activation_max, &op_params); + op_params.input1_offset = -data->input1_zero_point; + op_params.input2_offset = -data->input2_zero_point; + op_params.output_offset = data->output_zero_point; + op_params.output_multiplier = data->output_multiplier; + op_params.output_shift = data->output_shift; + + bool requires_broadcast = reference_ops::ProcessBroadcastShapes( + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorShape(input2), &op_params); + + if (requires_broadcast) { + TF_LITE_DIV(reference_ops, BroadcastDivSlow, int8_t); + } else { + TF_LITE_DIV(reference_ops, Div, int8_t); + } +#undef TF_LITE_DIV + } else { + TF_LITE_KERNEL_LOG( + context, "Unsupported combination of input and output types in DIV."); + return kTfLiteError; + } + + return kTfLiteOk; +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->builtin_data != nullptr); + auto* params = static_cast(node->builtin_data); + TFLITE_DCHECK(node->user_data != nullptr); + auto* data = static_cast(node->user_data); + + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + if (output->type == kTfLiteFloat32) { + EvalDiv(context, node, params, data, input1, input2, output); + } else if (output->type == kTfLiteInt8) { + TF_LITE_ENSURE_OK(context, EvalQuantized(context, node, params, data, + input1, input2, output)); + } else { + TF_LITE_KERNEL_LOG(context, + "DIV only supports FLOAT32, quantized INT8 " + "now, got type %s (%d).", + TfLiteTypeGetName(output->type), output->type); + return kTfLiteError; + } + + return kTfLiteOk; +} + +} // namespace + +TfLiteRegistration Register_DIV() { + return tflite::micro::RegisterOp(Init, Prepare, Eval); +} + +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/elementwise.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/elementwise.cc index 366dd610..b1cb1dcb 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/elementwise.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/elementwise.cc @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ limitations under the License. #include #include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/micro/kernels/kernel_util.h" @@ -27,6 +29,22 @@ namespace micro { namespace elementwise { namespace { +constexpr int kAbsNameId = 0; +constexpr int kRsrqtNameId = 1; + +const int kElementwiseInputTensor = 0; +const int kElementwiseOutputTensor = 0; + +struct OpDataAbsRsqrt { + int32_t multiplier; + int shift; + int input_offset; + int output_offset; + bool needs_rescale; + TfLiteQuantizationType input_quantization_type; + TfLiteType input_type; +}; + bool IsNumericSupportedType(const TfLiteType type) { return type == kTfLiteFloat32; } @@ -35,16 +53,40 @@ bool IsLogicalSupportedType(const TfLiteType type) { return type == kTfLiteBool; } +bool IsAbsSupportedType(const TfLiteType type) { + return type == kTfLiteFloat32 || type == kTfLiteInt8 || type == kTfLiteInt16; +} + +bool IsRsqrtSupportedType(const TfLiteType type) { + return type == kTfLiteFloat32 || type == kTfLiteInt8; +} + +inline void SetAbsOutputMultiplier(const float input_scale, + const float output_scale, + int32_t* multiplier, int* shift) { + QuantizeMultiplier(static_cast(input_scale / output_scale), + multiplier, shift); +} + +inline void SetRsqrtOutputMultiplier(const float input_scale, + const float output_scale, + int32_t* multiplier, int* shift) { + const double scale = + 1. / static_cast((std::sqrt(input_scale) * output_scale)); + QuantizeMultiplier(scale, multiplier, shift); +} + typedef bool (*IsSupportedType)(TfLiteType); template TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { MicroContext* micro_context = GetMicroContext(context); - TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); - TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); + TfLiteTensor* input = + micro_context->AllocateTempInputTensor(node, kElementwiseInputTensor); TF_LITE_ENSURE(context, input != nullptr); - TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); + TfLiteTensor* output = + micro_context->AllocateTempOutputTensor(node, kElementwiseOutputTensor); TF_LITE_ENSURE(context, output != nullptr); TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); if (!IsSupportedType(input->type)) { @@ -58,9 +100,79 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } +typedef bool (*IsSupportedType)(TfLiteType); +template +TfLiteStatus PrepareAbsRsqrt(TfLiteContext* context, TfLiteNode* node) { + MicroContext* micro_context = GetMicroContext(context); + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); + TF_LITE_ENSURE(context, input != nullptr); + TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); + TF_LITE_ENSURE(context, output != nullptr); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); + if (!IsSupportedType(input->type)) { + TF_LITE_KERNEL_LOG(context, "Input data type %s (%d) is not supported.", + TfLiteTypeGetName(input->type), input->type); + return kTfLiteError; + } + + auto* op_data = static_cast(node->user_data); + op_data->input_type = input->type; + + // For int16 type input, we support both quantized and non-quantized + // evaluation. + if (op_nameid == kAbsNameId) { + op_data->input_quantization_type = input->quantization.type; + } + + if (input->type == kTfLiteInt8 || + (input->type == kTfLiteInt16 && + input->quantization.type != kTfLiteNoQuantization)) { + TF_LITE_ENSURE_EQ(context, input->quantization.type, + kTfLiteAffineQuantization); + TF_LITE_ENSURE_EQ(context, output->quantization.type, + kTfLiteAffineQuantization); + const auto* input_params = + reinterpret_cast(input->quantization.params); + const auto* output_params = reinterpret_cast( + output->quantization.params); + TF_LITE_ENSURE(context, input_params != nullptr); + TF_LITE_ENSURE(context, input_params->scale != nullptr); + TF_LITE_ENSURE(context, input_params->scale->size > 0); + TF_LITE_ENSURE(context, input_params->zero_point->size > 0); + TF_LITE_ENSURE(context, output_params != nullptr); + TF_LITE_ENSURE(context, output_params->scale != nullptr); + TF_LITE_ENSURE(context, output_params->scale->size > 0); + TF_LITE_ENSURE(context, output_params->zero_point->size > 0); + op_data->input_offset = input_params->zero_point->data[0]; + op_data->output_offset = output_params->zero_point->data[0]; + if (input->type == kTfLiteInt16) { + TF_LITE_ENSURE_EQ(context, op_data->input_offset, 0); + TF_LITE_ENSURE_EQ(context, op_data->output_offset, 0); + } + const float input_scale = input_params->scale->data[0]; + const float output_scale = output_params->scale->data[0]; + op_data->needs_rescale = input_scale != output_scale; + if (op_nameid == kAbsNameId && op_data->needs_rescale) { + SetAbsOutputMultiplier(input_scale, output_scale, &op_data->multiplier, + &op_data->shift); + } else if (op_nameid == kRsrqtNameId) { + SetRsqrtOutputMultiplier(input_scale, output_scale, &op_data->multiplier, + &op_data->shift); + } + } + micro_context->DeallocateTempTfLiteTensor(input); + micro_context->DeallocateTempTfLiteTensor(output); + return kTfLiteOk; +} + template -inline TfLiteStatus EvalImpl(TfLiteContext* context, TfLiteNode* node, - T func(T), TfLiteType expected_type) { +inline TfLiteStatus EvalImplQuantized( + TfLiteContext* context, TfLiteNode* node, + T func(TfLiteContext*, TfLiteNode*, T), + TfLiteStatus validate_input_func(TfLiteContext*, TfLiteNode*, T), + TfLiteType expected_type) { const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); TF_LITE_ENSURE_TYPES_EQ(context, input->type, expected_type); @@ -68,6 +180,34 @@ inline TfLiteStatus EvalImpl(TfLiteContext* context, TfLiteNode* node, const T* in_data = tflite::micro::GetTensorData(input); T* out_data = tflite::micro::GetTensorData(output); for (size_t i = 0; i < num_elements; ++i) { + if (validate_input_func) { + TF_LITE_ENSURE_OK(context, + validate_input_func(context, node, in_data[i])); + } + out_data[i] = func(context, node, in_data[i]); + } + return kTfLiteOk; +} + +template +inline T AbsHelper(T i) { + return std::abs(i); +} + +template +inline TfLiteStatus EvalImpl(TfLiteContext* context, TfLiteNode* node, + T func(T), TfLiteStatus validate_input_func(T), + TfLiteType expected_type) { + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, expected_type); + const size_t num_elements = ElementCount(*input->dims); + const T* in_data = tflite::micro::GetTensorData(input); + T* out_data = tflite::micro::GetTensorData(output); + for (size_t i = 0; i < num_elements; ++i) { + if (validate_input_func) { + TF_LITE_ENSURE_OK(context, validate_input_func(in_data[i])); + } out_data[i] = func(in_data[i]); } return kTfLiteOk; @@ -75,16 +215,114 @@ inline TfLiteStatus EvalImpl(TfLiteContext* context, TfLiteNode* node, inline TfLiteStatus EvalNumeric(TfLiteContext* context, TfLiteNode* node, float float_func(float)) { - return EvalImpl(context, node, float_func, kTfLiteFloat32); + return EvalImpl(context, node, float_func, + /*validate_input_func=*/nullptr, kTfLiteFloat32); } inline TfLiteStatus EvalLogical(TfLiteContext* context, TfLiteNode* node, + bool bool_func(bool)) { - return EvalImpl(context, node, bool_func, kTfLiteBool); + return EvalImpl(context, node, bool_func, + /*validate_input_func=*/nullptr, kTfLiteBool); +} + +void* ElementWiseAbsRsqrtInit(TfLiteContext* context, const char* buffer, + size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpDataAbsRsqrt)); +} + +template +inline T AbsEvalQuantized(TfLiteContext* context, TfLiteNode* node, T i) { + const auto* op_data = static_cast(node->user_data); + const int kMin = std::numeric_limits::min(); + const int kMax = std::numeric_limits::max(); + + const int32_t value = std::abs(i - op_data->input_offset); + if (!op_data->needs_rescale) { + return static_cast( + std::min(std::max(static_cast(value + op_data->output_offset), + static_cast(kMin)), + static_cast(kMax))); + } + + const int32_t output = tflite::MultiplyByQuantizedMultiplier( + value, op_data->multiplier, op_data->shift) + + op_data->output_offset; + return static_cast(std::min( + std::max(static_cast(output), static_cast(kMin)), + static_cast(kMax))); +} + +template +inline T RsqrtEvalQuantized(TfLiteContext* context, TfLiteNode* node, T i) { + const auto* op_data = static_cast(node->user_data); + const int kMin = std::numeric_limits::min(); + const int kMax = std::numeric_limits::max(); + + const int32_t value = (i - op_data->input_offset); + const int32_t kShift = 20; // Shift to keep value integer. + if (value == 0) { + // Assume that any value close to 0 represents the max output value. + return static_cast(kMax); + } + int32_t inv_sqrt_multiplier; + int inv_sqrt_shift; + GetInvSqrtQuantizedMultiplierExp(value, kReverseShift, &inv_sqrt_multiplier, + &inv_sqrt_shift); + const int32_t data = tflite::MultiplyByQuantizedMultiplier( + static_cast(1), inv_sqrt_multiplier, inv_sqrt_shift + kShift); + const int32_t output = + tflite::MultiplyByQuantizedMultiplier(data, op_data->multiplier, + op_data->shift - kShift) + + op_data->output_offset; + return static_cast(std::min( + std::max(static_cast(output), static_cast(kMin)), + static_cast(kMax))); +} + +template +TfLiteStatus validate_input_func(TfLiteContext* context, TfLiteNode* node, + T i) { + const auto* op_data = static_cast(node->user_data); + + TF_LITE_ENSURE_MSG(context, i >= op_data->input_offset, + "Rsqrt is only defined for positive values"); + return static_cast(kTfLiteOk); } TfLiteStatus AbsEval(TfLiteContext* context, TfLiteNode* node) { - return EvalNumeric(context, node, std::abs); + OpDataAbsRsqrt* op_data = reinterpret_cast(node->user_data); + TfLiteType type = op_data->input_type; + TfLiteQuantizationType input_quantization_type = + op_data->input_quantization_type; + TfLiteStatus eval_result; + + switch (type) { + case kTfLiteFloat32: + eval_result = EvalNumeric(context, node, std::abs); + break; + case kTfLiteInt8: + eval_result = + EvalImplQuantized(context, node, AbsEvalQuantized, + /*validate_input_func=*/nullptr, type); + break; + case kTfLiteInt16: + eval_result = + input_quantization_type == kTfLiteNoQuantization + ? EvalImpl(context, node, AbsHelper, + /*validate_input_func=*/nullptr, type) + : EvalImplQuantized(context, node, AbsEvalQuantized, + /*validate_input_func=*/nullptr, + type); + break; + default: + TF_LITE_KERNEL_LOG(context, "Current data type %s is not supported.", + TfLiteTypeGetName(type)); + return kTfLiteError; + break; + } + return eval_result; } TfLiteStatus SinEval(TfLiteContext* context, TfLiteNode* node) { @@ -104,7 +342,23 @@ TfLiteStatus SqrtEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus RsqrtEval(TfLiteContext* context, TfLiteNode* node) { - return EvalNumeric(context, node, [](float f) { return 1.f / std::sqrt(f); }); + const auto* op_data = static_cast(node->user_data); + TfLiteType type = op_data->input_type; + switch (type) { + case kTfLiteFloat32: + return EvalImpl( + context, node, [](float f) { return 1.f / std::sqrt(f); }, + /*validate_input_func=*/nullptr, type); + case kTfLiteInt8: + return EvalImplQuantized(context, node, + elementwise::RsqrtEvalQuantized, + elementwise::validate_input_func, type); + + default: + TF_LITE_KERNEL_LOG(context, "Current data type %s is not supported.", + TfLiteTypeGetName(type)); + return kTfLiteError; + } } TfLiteStatus SquareEval(TfLiteContext* context, TfLiteNode* node) { @@ -119,101 +373,57 @@ TfLiteStatus LogicalNotEval(TfLiteContext* context, TfLiteNode* node) { } // namespace elementwise TfLiteRegistration Register_ABS() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::AbsEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + elementwise::ElementWiseAbsRsqrtInit, + elementwise::PrepareAbsRsqrt, + elementwise::AbsEval); } TfLiteRegistration Register_SIN() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::SinEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + nullptr, elementwise::GenericPrepare, + elementwise::SinEval); } TfLiteRegistration Register_COS() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::CosEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + nullptr, elementwise::GenericPrepare, + elementwise::CosEval); } TfLiteRegistration Register_LOG() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::LogEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + nullptr, elementwise::GenericPrepare, + elementwise::LogEval); } TfLiteRegistration Register_SQRT() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::SqrtEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + nullptr, elementwise::GenericPrepare, + elementwise::SqrtEval); } TfLiteRegistration Register_RSQRT() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::RsqrtEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + elementwise::ElementWiseAbsRsqrtInit, + elementwise::PrepareAbsRsqrt, + elementwise::RsqrtEval); } TfLiteRegistration Register_SQUARE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::SquareEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + nullptr, elementwise::GenericPrepare, + elementwise::SquareEval); } TfLiteRegistration Register_LOGICAL_NOT() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/ - elementwise::GenericPrepare, - /*invoke=*/elementwise::LogicalNotEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + nullptr, elementwise::GenericPrepare, + elementwise::LogicalNotEval); } } // namespace micro } // namespace ops -} // namespace tflite +} // namespace tflite \ No newline at end of file diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/elu.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/elu.cc index b2cd19cc..0b64e89d 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/elu.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/elu.cc @@ -146,14 +146,7 @@ TfLiteStatus EluEval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_ELU() { - return {/*init=*/EluInit, - /*free=*/nullptr, - /*prepare=*/EluPrepare, - /*invoke=*/EluEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(EluInit, EluPrepare, EluEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/add.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/add.cc index 47a17d9f..2f1ac58d 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/add.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/add.cc @@ -196,14 +196,7 @@ TfLiteStatus AddEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration Register_ADD() { - return {/*init=*/AddInit, - /*free=*/nullptr, - /*prepare=*/AddPrepare, - /*invoke=*/AddEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(AddInit, AddPrepare, AddEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/conv.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/conv.cc index 09260482..919dd006 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/conv.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/conv.cc @@ -112,9 +112,24 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { #if ESP_NN if (input->type == kTfLiteInt8) { + data_dims_t input_dims = { + .width = input_width, .height = input_height, + .channels = input->dims->data[3], 1 + }; + data_dims_t output_dims = { + .width = output_width, .height = output_height, + .channels = output->dims->data[3], 1 + }; + data_dims_t filter_dims = {.width = filter_width, .height = filter_height, 0, 0}; + conv_params_t conv_params = { + .in_offset = 0, .out_offset = 0, + .stride = {params.stride_width, params.stride_height}, + .padding = {data->op_data.padding.width, data->op_data.padding.height}, + .dilation = {0, 0}, .activation = {-128, 127} + }; + int scratch_buf_size = esp_nn_get_conv_scratch_size( - input_width, input_height, input->dims->data[3], - output->dims->data[3], filter_width, filter_height); + &input_dims, &filter_dims, &output_dims, &conv_params); if (scratch_buf_size > 0) { TF_LITE_ENSURE_STATUS(context->RequestScratchBufferInArena( context, scratch_buf_size, &data->buffer_idx)); @@ -191,18 +206,33 @@ inline void EvalQuantizedPerChannel( const int input_size = input_width * input_height * input_depth; const int output_size = output_width * output_height * output_depth; + data_dims_t input_dims = { + .width = input_width, .height = input_height, + .channels = input_depth, 1 + }; + data_dims_t output_dims = { + .width = output_width, .height = output_height, + .channels = output_depth, 1 + }; + data_dims_t filter_dims = {.width = filter_width, .height = filter_height, 0, 0}; + conv_params_t conv_params = { + .in_offset = input_offset, .out_offset = output_offset, + .stride = {stride_width, stride_height}, + .padding = {pad_width, pad_height}, + .dilation = {0, 0}, + .activation = {activation_min, activation_max} + }; + quant_data_t quant_data = { + .shift = data.op_data.per_channel_output_shift, + .mult = data.op_data.per_channel_output_multiplier + }; + for (int i_batch = 0; i_batch < batch_size; i_batch++) { - esp_nn_conv_s8(input_data + i_batch * input_size, - input_width, input_height, input_depth, input_offset, - pad_width, pad_height, stride_width, stride_height, - tflite::micro::GetTensorData(filter), - filter_width, filter_height, + esp_nn_conv_s8(&input_dims, input_data + i_batch * input_size, + &filter_dims, tflite::micro::GetTensorData(filter), tflite::micro::GetTensorData(bias), - output_data + i_batch * output_size, - output_width, output_height, output_depth, output_offset, - data.op_data.per_channel_output_shift, - data.op_data.per_channel_output_multiplier, - activation_min, activation_max); + &output_dims, output_data + i_batch * output_size, + &conv_params, &quant_data); } } else { reference_integer_ops::ConvPerChannel( @@ -299,21 +329,16 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TfLiteTypeGetName(input->type), input->type); return kTfLiteError; } - conv_total_time += esp_timer_get_time() - start_time; + long long time_this_instance = esp_timer_get_time() - start_time; + conv_total_time += time_this_instance; + //printf("time this instance: %llu\n", time_this_instance / 1000); return kTfLiteOk; } } // namespace TfLiteRegistration Register_CONV_2D() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/depthwise_conv.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/depthwise_conv.cc index 5f2d9d50..a2460248 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/depthwise_conv.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/depthwise_conv.cc @@ -112,21 +112,36 @@ inline void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, if (data.buffer_idx > -1) { scratch_buf = context->GetScratchBuffer(context, data.buffer_idx); } + esp_nn_set_depthwise_conv_scratch_buf(scratch_buf); + data_dims_t input_dims = { + .width = input_width, .height = input_height, + .channels = input_depth, 1 + }; + data_dims_t output_dims = { + .width = output_width, .height = output_height, + .channels = output_depth, 1 + }; + data_dims_t filter_dims = {.width = filter_width, .height = filter_height, 0, 0}; + dw_conv_params_t conv_params = { + .in_offset = input_offset, .out_offset = output_offset, + .ch_mult = depth_multiplier, + .stride = {stride_width, stride_height}, + .padding = {pad_width, pad_height}, .dilation = {0, 0}, + .activation = {activation_min, activation_max} + }; + quant_data_t quant_data = { + .shift = data.op_data.per_channel_output_shift, + .mult = data.op_data.per_channel_output_multiplier + }; + for (int i_batch = 0; i_batch < batch_size; i_batch++) { - esp_nn_depthwise_conv_s8(input_data + i_batch * input_size, input_width, - input_height, input_depth, input_offset, - pad_width, pad_height, - stride_width, stride_height, depth_multiplier, - tflite::micro::GetTensorData(filter), - filter_width, filter_height, + esp_nn_depthwise_conv_s8(&input_dims, input_data + i_batch * input_size, + &filter_dims, tflite::micro::GetTensorData(filter), tflite::micro::GetTensorData(bias), - output_data + i_batch * output_size, - output_width, output_height, output_offset, - data.op_data.per_channel_output_shift, - data.op_data.per_channel_output_multiplier, - activation_min, activation_max); + &output_dims, output_data + i_batch * output_size, + &conv_params, &quant_data); } } else { reference_integer_ops::DepthwiseConvPerChannel( @@ -209,9 +224,25 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { #if ESP_NN if (input->type == kTfLiteInt8) { + data_dims_t input_dims = { + .width = input_width, .height = input_height, + .channels = input->dims->data[3], 1 + }; + data_dims_t output_dims = { + .width = output_width, .height = output_height, + .channels = output->dims->data[3], 1 + }; + data_dims_t filter_dims = {.width = filter_width, .height = filter_height, 0, 0}; + dw_conv_params_t conv_params = { + .in_offset = 0, .out_offset = 0, + .ch_mult = params.depth_multiplier, + .stride = {params.stride_width, params.stride_height}, + .padding = {data->op_data.padding.width, data->op_data.padding.height}, + .dilation = {0, 0}, .activation = {-128, 127} + }; + int scratch_buf_size = esp_nn_get_depthwise_conv_scratch_size( - input_width, input_height, input->dims->data[3], - params.depth_multiplier, filter_width, filter_height); + &input_dims, &filter_dims, &output_dims, &conv_params); if (scratch_buf_size > 0) { TF_LITE_ENSURE_STATUS(context->RequestScratchBufferInArena( context, scratch_buf_size, &data->buffer_idx)); @@ -299,21 +330,17 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TfLiteTypeGetName(input->type), input->type); return kTfLiteError; } - dc_total_time += esp_timer_get_time() - start_time; + long long time_this_instance = esp_timer_get_time() - start_time; + dc_total_time += time_this_instance; + // printf("time this instance: %llu\n", time_this_instance / 1000); + return kTfLiteOk; } } // namespace TfLiteRegistration Register_DEPTHWISE_CONV_2D() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/fully_connected.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/fully_connected.cc index 5e1705da..484cffb6 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/fully_connected.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/fully_connected.cc @@ -185,14 +185,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_FULLY_CONNECTED() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/mul.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/mul.cc index 0e8a82f4..02413f5c 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/mul.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/mul.cc @@ -118,14 +118,7 @@ TfLiteStatus MulEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration Register_MUL() { - return {/*init=*/MulInit, - /*free=*/nullptr, - /*prepare=*/MulPrepare, - /*invoke=*/MulEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(MulInit, MulPrepare, MulEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/pooling.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/pooling.cc index d55bab82..b450929e 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/pooling.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/pooling.cc @@ -221,25 +221,11 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) { } // namespace TfLiteRegistration Register_AVERAGE_POOL_2D() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/PoolingPrepare, - /*invoke=*/AverageEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, PoolingPrepare, AverageEval); } TfLiteRegistration Register_MAX_POOL_2D() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/PoolingPrepare, - /*invoke=*/MaxEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, PoolingPrepare, MaxEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/softmax.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/softmax.cc new file mode 100644 index 00000000..9a967839 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/esp_nn/softmax.cc @@ -0,0 +1,208 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/kernels/softmax.h" + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/softmax.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" + +#include "freertos/FreeRTOS.h" +#include + +#if ESP_NN +#include +#endif + +long long softmax_total_time = 0; + +namespace tflite { +namespace { +// Softmax parameter data that persists in user_data +const int kInt16LUTArraySize = 513; + +struct NodeData { + SoftmaxParams op_data; +#if ESP_NN + int buffer_idx; +#endif +}; + +static void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(NodeData)); +} + +void SoftmaxQuantized(TfLiteContext* context, const TfLiteEvalTensor* input, + TfLiteEvalTensor* output, const NodeData* data) { + if (input->type == kTfLiteInt8) { + if (output->type == kTfLiteInt16) { + tflite::reference_ops::Softmax( + data->op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } else { +#if ESP_NN + const int32_t input_beta_multiplier = data->op_data.input_multiplier; + const int32_t input_beta_left_shift = data->op_data.input_left_shift; + const int diff_min = data->op_data.diff_min; + const RuntimeShape input_shape = tflite::micro::GetTensorShape(input); + const RuntimeShape output_shape = tflite::micro::GetTensorShape(output); + const int trailing_dim = input_shape.DimensionsCount() - 1; + const int outer_size = + MatchingFlatSizeSkipDim(input_shape, trailing_dim, output_shape); + const int depth = + MatchingDim(input_shape, trailing_dim, output_shape, trailing_dim); + const int8_t *in_ptr = tflite::micro::GetTensorData(input); + int8_t *out_ptr = tflite::micro::GetTensorData(output); + void *scratch_buf = NULL; + if (data->buffer_idx > -1) { + scratch_buf = context->GetScratchBuffer(context, data->buffer_idx); + } + esp_nn_set_softmax_scratch_buf(scratch_buf); + esp_nn_softmax_s8(in_ptr, outer_size, depth, input_beta_multiplier, + input_beta_left_shift, diff_min, out_ptr); +#else + tflite::reference_ops::Softmax( + data->op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +#endif + } + } else { + tflite::reference_ops::SoftmaxInt16( + data->op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } +} + +static TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); + + TFLITE_DCHECK(node->user_data != nullptr); + NodeData data = *static_cast(node->user_data); + + long long start_time = esp_timer_get_time(); + switch (input->type) { + case kTfLiteFloat32: { + tflite::reference_ops::Softmax( + data.op_data, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } + break; + case kTfLiteInt8: + case kTfLiteInt16: { + SoftmaxQuantized(context, input, output, &data); + } + break; + default: + TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", + TfLiteTypeGetName(input->type), input->type); + return kTfLiteError; + } + softmax_total_time += esp_timer_get_time() - start_time; + return kTfLiteOk; +} + +static TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + MicroContext* micro_context = GetMicroContext(context); + + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); + TF_LITE_ENSURE(context, input != nullptr); + TF_LITE_ENSURE(context, NumDimensions(input) >= 1); + TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); + TF_LITE_ENSURE(context, output != nullptr); + + TF_LITE_ENSURE(context, node->user_data != nullptr); + NodeData* data = static_cast(node->user_data); + // Only allocate LUTs for KTfLiteInt16 data type + if (input->type == kTfLiteInt16) { + void* raw_exp_lut = context->AllocatePersistentBuffer( + context, sizeof(int16_t) * kInt16LUTArraySize); + TF_LITE_ENSURE(context, raw_exp_lut != nullptr); + data->op_data.exp_lut = reinterpret_cast(raw_exp_lut); + void* one_over_one_plus_x_lut = context->AllocatePersistentBuffer( + context, sizeof(int16_t) * kInt16LUTArraySize); + TF_LITE_ENSURE(context, one_over_one_plus_x_lut != nullptr); + data->op_data.one_over_one_plus_x_lut = + reinterpret_cast(one_over_one_plus_x_lut); + } + + if (output->type == kTfLiteInt16) { + TF_LITE_ENSURE(context, + input->type == kTfLiteInt8 || input->type == kTfLiteInt16); + } else { + TF_LITE_ENSURE_EQ(context, input->type, output->type); + } + + // Populate LUT if required + if (input->type == kTfLiteInt16) { + TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); + // exp LUT only used on negative values + // we consider exp(-10.0) is insignificant to accumulation + gen_lut( + [](float value) { return std::exp(value); }, -10.0f, 0.0f, -1.0f, 1.0f, + data->op_data.exp_lut); + gen_lut( + [](float value) { return 1.0f / (1.0f + value); }, 0.0f, 1.0f, -1.0f, + 1.0f, data->op_data.one_over_one_plus_x_lut); + data->op_data.zero_point = output->params.zero_point; + data->op_data.scale = output->params.scale; + } + + auto* params = static_cast(node->builtin_data); + auto ret_val = + CalculateSoftmaxParams(context, input, output, params, &data->op_data); + +#if ESP_NN + if (output->type == kTfLiteInt8 && input->type == kTfLiteInt8) { + const int32_t input_width = input->dims->data[1]; + const int32_t input_height = input->dims->data[2]; + int scratch_buf_size = esp_nn_get_softmax_scratch_size(input_width, + input_height); + if (scratch_buf_size > 0) { + TF_LITE_ENSURE_STATUS(context->RequestScratchBufferInArena( + context, scratch_buf_size, &data->buffer_idx)); + } + } +#endif + + micro_context->DeallocateTempTfLiteTensor(input); + micro_context->DeallocateTempTfLiteTensor(output); + return ret_val; +} + +} // namespace + +TfLiteRegistration Register_SOFTMAX() { + return tflite::micro::RegisterOp(Init, Prepare, Eval); +} + +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/exp.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/exp.cc index d1b0f6cb..ae26f636 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/exp.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/exp.cc @@ -72,14 +72,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_EXP() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/expand_dims.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/expand_dims.cc index 6dcba4d5..4b105bf6 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/expand_dims.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/expand_dims.cc @@ -146,14 +146,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_EXPAND_DIMS() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/fill.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/fill.cc index d8a2b09d..9f438b89 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/fill.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/fill.cc @@ -135,14 +135,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_FILL() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor.cc index b8be1cf0..6b2a4cc2 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor.cc @@ -42,14 +42,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace floor TfLiteRegistration Register_FLOOR() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/floor::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, nullptr, floor::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor_div.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor_div.cc index d11e4969..333a1eba 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor_div.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor_div.cc @@ -123,14 +123,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_FLOOR_DIV() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor_mod.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor_mod.cc index 083bd5cb..9bb49497 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor_mod.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/floor_mod.cc @@ -121,14 +121,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_FLOOR_MOD() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/fully_connected.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/fully_connected.cc index c0be3814..a083edd7 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/fully_connected.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/fully_connected.cc @@ -1,4 +1,4 @@ -/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -55,10 +55,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TfLiteTensor* output = micro_context->AllocateTempOutputTensor( node, kFullyConnectedOutputTensor); TF_LITE_ENSURE(context, output != nullptr); - TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); - TF_LITE_ENSURE_MSG(context, input->type == filter->type, - "Hybrid models are not supported on TFLite Micro."); TF_LITE_ENSURE_OK(context, CalculateOpDataFullyConnected( context, params->activation, input->type, @@ -126,6 +123,23 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { break; } + case kTfLiteInt16: { + const int64_t* bias_data = + nullptr != bias ? tflite::micro::GetTensorData(bias) + : nullptr; + + tflite::reference_integer_ops::FullyConnected( + FullyConnectedParamsQuantized(data), + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(filter), + tflite::micro::GetTensorData(filter), + tflite::micro::GetTensorShape(bias), bias_data, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + break; + } + default: { TF_LITE_KERNEL_LOG(context, "Type %s (%d) not supported.", TfLiteTypeGetName(input->type), input->type); @@ -138,14 +152,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_FULLY_CONNECTED() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/fully_connected.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/fully_connected.h index e1215da6..93026cd5 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/fully_connected.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/fully_connected.h @@ -1,4 +1,4 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -81,6 +81,24 @@ inline TfLiteRegistration Register_FULLY_CONNECTED_INT8() { } #endif + +#if defined(CMSIS_NN) +// Returns a TfLiteRegistration struct for kernel variant that only supports +// int16. +TfLiteRegistration Register_FULLY_CONNECTED_INT16(); + +#else +// Note that while this block gets used for both reference and optimized kernels +// that do not have any specialized implementations, the only goal here is to +// define fallback implementation that allow reference kernels to still be used +// from applications that call a more specific kernel variant. + +inline TfLiteRegistration Register_FULLY_CONNECTED_INT16() { + return Register_FULLY_CONNECTED(); +} + +#endif + } // namespace tflite #endif // TENSORFLOW_LITE_MICRO_KERNELS_FULLY_CONNECTED_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/gather.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/gather.cc index 0b7c23f9..6035efa7 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/gather.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/gather.cc @@ -218,14 +218,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_GATHER() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/gather_nd.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/gather_nd.cc index c604ae15..4327177d 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/gather_nd.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/gather_nd.cc @@ -131,7 +131,8 @@ TfLiteStatus GatherNd(const TfLiteEvalTensor* params, slice_size *= params->dims->data[i]; } - int remain_flat_size = ElementCount(*params->dims); + int params_flat_size = ElementCount(*params->dims); + int remain_flat_size = params_flat_size; // Number of elements per dimension int dims_to_count[MAX_INDICES_ND]; @@ -147,6 +148,9 @@ TfLiteStatus GatherNd(const TfLiteEvalTensor* params, IndicesT index = index_data[offset]; from_pos += index * dims_to_count[j]; } + if (from_pos < 0 || from_pos + slice_size > params_flat_size) { + return kTfLiteError; + } std::memcpy(output_data + i * slice_size, param_data + from_pos, sizeof(ParamsT) * slice_size); } @@ -158,12 +162,13 @@ TfLiteStatus EvalGatherNd(TfLiteContext* context, const TfLiteEvalTensor* params, const TfLiteEvalTensor* indices, TfLiteEvalTensor* output) { + TfLiteStatus status = kTfLiteError; switch (params->type) { case kTfLiteFloat32: - return GatherNd(params, indices, output); + status = GatherNd(params, indices, output); break; case kTfLiteInt8: - return GatherNd(params, indices, output); + status = GatherNd(params, indices, output); break; default: TF_LITE_KERNEL_LOG(context, @@ -171,6 +176,10 @@ TfLiteStatus EvalGatherNd(TfLiteContext* context, TfLiteTypeGetName(params->type)); return kTfLiteError; } + if (status != kTfLiteOk) { + TF_LITE_KERNEL_LOG(context, "gather_nd index out of bounds"); + } + return status; } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { @@ -195,14 +204,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_GATHER_ND() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/hard_swish.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/hard_swish.cc index 060dfc14..055e12e6 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/hard_swish.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/hard_swish.cc @@ -68,14 +68,8 @@ TfLiteStatus HardSwishEval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_HARD_SWISH() { - return {/*init=*/HardSwishInit, - /*free=*/nullptr, - /*prepare=*/tflite::HardSwishPrepare, - /*invoke=*/HardSwishEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(HardSwishInit, tflite::HardSwishPrepare, + HardSwishEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/if.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/if.cc index 050aeac4..39eca8b4 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/if.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/if.cc @@ -115,14 +115,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace. TfLiteRegistration Register_IF() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_runner.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_runner.cc index fd84dec1..dd4c14f4 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_runner.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_runner.cc @@ -15,9 +15,9 @@ limitations under the License. #include "tensorflow/lite/micro/kernels/kernel_runner.h" +#include "tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h" #include "tensorflow/lite/micro/micro_arena_constants.h" #include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/simple_memory_allocator.h" #include "tensorflow/lite/micro/test_helpers.h" namespace tflite { @@ -27,14 +27,21 @@ namespace micro { constexpr int KernelRunner::kKernelRunnerBufferSize_; uint8_t KernelRunner::kKernelRunnerBuffer_[]; +void ClearBufferApi(TfLiteContext* context_) { + context_->GetScratchBuffer = nullptr; + context_->GetExternalContext = nullptr; + context_->AllocatePersistentBuffer = nullptr; + context_->RequestScratchBufferInArena = nullptr; +} + KernelRunner::KernelRunner(const TfLiteRegistration& registration, TfLiteTensor* tensors, int tensors_size, TfLiteIntArray* inputs, TfLiteIntArray* outputs, - void* builtin_data) + void* builtin_data, TfLiteIntArray* intermediates) : registration_(registration), - allocator_(SimpleMemoryAllocator::Create(GetMicroErrorReporter(), - kKernelRunnerBuffer_, - kKernelRunnerBufferSize_)), + allocator_(SingleArenaBufferAllocator::Create(GetMicroErrorReporter(), + kKernelRunnerBuffer_, + kKernelRunnerBufferSize_)), mock_micro_graph_(allocator_), fake_micro_context_(tensors, allocator_, &mock_micro_graph_) { // Prepare TfLiteContext: @@ -43,10 +50,8 @@ KernelRunner::KernelRunner(const TfLiteRegistration& registration, context_.recommended_num_threads = 1; context_.GetTensor = MicroContextGetTensor; context_.GetEvalTensor = MicroContextGetEvalTensor; + tflite::micro::ClearBufferApi(&context_); context_.AllocatePersistentBuffer = MicroContextAllocatePersistentBuffer; - context_.RequestScratchBufferInArena = - MicroContextRequestScratchBufferInArena; - context_.GetScratchBuffer = MicroContextGetScratchBuffer; context_.recommended_num_threads = 0; @@ -54,6 +59,7 @@ KernelRunner::KernelRunner(const TfLiteRegistration& registration, node_.inputs = inputs; node_.outputs = outputs; node_.builtin_data = builtin_data; + node_.intermediates = intermediates; } bool KernelRunner::ValidateTempBufferDeallocated() { @@ -63,12 +69,19 @@ bool KernelRunner::ValidateTempBufferDeallocated() { TfLiteStatus KernelRunner::InitAndPrepare(const char* init_data, size_t length) { if (registration_.init) { + tflite::micro::ClearBufferApi(&context_); + context_.AllocatePersistentBuffer = MicroContextAllocatePersistentBuffer; node_.user_data = registration_.init(&context_, init_data, length); } TF_LITE_ENSURE(&context_, ValidateTempBufferDeallocated()); if (registration_.prepare) { + tflite ::micro::ClearBufferApi(&context_); + context_.AllocatePersistentBuffer = MicroContextAllocatePersistentBuffer; + context_.RequestScratchBufferInArena = + MicroContextRequestScratchBufferInArena; + context_.GetExternalContext = MicroContextGetExternalContext; TF_LITE_ENSURE_STATUS(registration_.prepare(&context_, &node_)); } @@ -78,6 +91,9 @@ TfLiteStatus KernelRunner::InitAndPrepare(const char* init_data, } TfLiteStatus KernelRunner::Invoke() { + tflite::micro::ClearBufferApi(&context_); + context_.GetScratchBuffer = MicroContextGetScratchBuffer; + if (registration_.invoke == nullptr) { MicroPrintf("TfLiteRegistration missing invoke function pointer!"); return kTfLiteError; diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_runner.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_runner.h index 9dddde50..4482b70e 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_runner.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_runner.h @@ -18,9 +18,9 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h" #include "tensorflow/lite/micro/fake_micro_context.h" #include "tensorflow/lite/micro/mock_micro_graph.h" -#include "tensorflow/lite/micro/simple_memory_allocator.h" namespace tflite { namespace micro { @@ -35,7 +35,8 @@ class KernelRunner { public: KernelRunner(const TfLiteRegistration& registration, TfLiteTensor* tensors, int tensors_size, TfLiteIntArray* inputs, - TfLiteIntArray* outputs, void* builtin_data); + TfLiteIntArray* outputs, void* builtin_data, + TfLiteIntArray* intermediates = nullptr); // Calls init and prepare on the kernel (i.e. TfLiteRegistration) struct. Any // exceptions will be DebugLog'd and returned as a status code. @@ -64,7 +65,7 @@ class KernelRunner { TfLiteNode node_ = {}; const TfLiteRegistration& registration_; - SimpleMemoryAllocator* allocator_; + SingleArenaBufferAllocator* allocator_; MockMicroGraph mock_micro_graph_; FakeMicroContext fake_micro_context_; }; diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_util.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_util.cc index 14664b91..91c0bc91 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_util.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_util.cc @@ -36,6 +36,21 @@ int ValidateTensorIndexing(const TfLiteContext* context, int index, } // namespace +TfLiteRegistration RegisterOp( + void* (*init)(TfLiteContext* context, const char* buffer, size_t length), + TfLiteStatus (*prepare)(TfLiteContext* context, TfLiteNode* node), + TfLiteStatus (*invoke)(TfLiteContext* context, TfLiteNode* node)) { + return {/*init=*/init, + /*free=*/nullptr, + /*prepare=*/prepare, + /*invoke=*/invoke, + /*profiling_string=*/nullptr, + /*builtin_code=*/0, + /*custom_name=*/nullptr, + /*version=*/0, + /*registration_external=*/nullptr}; +} + // Returns a mutable tensor for a given input index. is_variable must be checked // during prepare when the full TfLiteTensor is available. TfLiteEvalTensor* GetMutableEvalInput(const TfLiteContext* context, diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_util.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_util.h index 6c5d7f18..d6f20c72 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_util.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/kernel_util.h @@ -27,6 +27,11 @@ limitations under the License. namespace tflite { namespace micro { +TfLiteRegistration RegisterOp( + void* (*init)(TfLiteContext* context, const char* buffer, size_t length), + TfLiteStatus (*prepare)(TfLiteContext* context, TfLiteNode* node), + TfLiteStatus (*invoke)(TfLiteContext* context, TfLiteNode* node)); + // Returns a mutable tensor for a given input index. is_variable must be checked // during prepare when the full TfLiteTensor is available. TfLiteEvalTensor* GetMutableEvalInput(const TfLiteContext* context, @@ -40,19 +45,33 @@ const TfLiteEvalTensor* GetEvalInput(const TfLiteContext* context, TfLiteEvalTensor* GetEvalOutput(const TfLiteContext* context, const TfLiteNode* node, int index); -// Returns data for a TfLiteEvalTensor struct. +// Returns data for a TfLiteEvalTensor struct that are expected to exist. template T* GetTensorData(TfLiteEvalTensor* tensor) { - return tensor != nullptr ? reinterpret_cast(tensor->data.raw) : nullptr; + TFLITE_DCHECK(tensor != nullptr); + return reinterpret_cast(tensor->data.raw); } -// Returns const data for a TfLiteEvalTensor struct. +// Returns const data for a TfLiteEvalTensor struct that are expected to exist. template const T* GetTensorData(const TfLiteEvalTensor* tensor) { TFLITE_DCHECK(tensor != nullptr); return reinterpret_cast(tensor->data.raw); } +// Returns data for a TfLiteEvalTensor struct that could be null. +template +T* GetOptionalTensorData(TfLiteEvalTensor* tensor) { + return tensor == nullptr ? nullptr : reinterpret_cast(tensor->data.raw); +} + +// Returns const data for a TfLiteEvalTensor struct that could be null. +template +const T* GetOptionalTensorData(const TfLiteEvalTensor* tensor) { + return tensor == nullptr ? nullptr + : reinterpret_cast(tensor->data.raw); +} + // Returns the shape of a TfLiteEvalTensor struct. const RuntimeShape GetTensorShape(const TfLiteEvalTensor* tensor); diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/l2_pool_2d.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/l2_pool_2d.cc index 250cd3be..2b2a27bf 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/l2_pool_2d.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/l2_pool_2d.cc @@ -136,14 +136,7 @@ TfLiteStatus L2Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_L2_POOL_2D() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/L2Prepare, - /*invoke=*/L2Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, L2Prepare, L2Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/l2norm.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/l2norm.cc index 289e4de5..45858e78 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/l2norm.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/l2norm.cc @@ -137,14 +137,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace l2norm TfLiteRegistration Register_L2NORM_REF() { - return {/*init=*/l2norm::Init, - /*free=*/nullptr, - /*prepare=*/l2norm::Prepare, - /*invoke=*/l2norm::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(l2norm::Init, l2norm::Prepare, l2norm::Eval); } TfLiteRegistration Register_L2_NORMALIZATION() { return Register_L2NORM_REF(); } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/leaky_relu.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/leaky_relu.cc index 70ee3856..96c1b1b1 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/leaky_relu.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/leaky_relu.cc @@ -88,14 +88,8 @@ TfLiteStatus LeakyReluEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration Register_LEAKY_RELU() { - return {/*init=*/LeakyReluInit, - /*free=*/nullptr, - /*prepare=*/LeakyReluPrepare, - /*invoke=*/LeakyReluEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(LeakyReluInit, LeakyReluPrepare, + LeakyReluEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/log_softmax.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/log_softmax.cc index 0af74def..5fd87612 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/log_softmax.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/log_softmax.cc @@ -142,14 +142,7 @@ TfLiteStatus LogSoftmaxEval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_LOG_SOFTMAX() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/LogSoftmaxPrepare, - /*invoke=*/LogSoftmaxEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, LogSoftmaxPrepare, LogSoftmaxEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/logical.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/logical.cc index e2d2b5f8..c85e0c5b 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/logical.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/logical.cc @@ -34,29 +34,11 @@ TfLiteStatus LogicalAndEval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_LOGICAL_OR() { - // Init, Free, Prepare, Eval are satisfying the Interface required by - // TfLiteRegistration. - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/LogicalOrEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, nullptr, LogicalOrEval); } TfLiteRegistration Register_LOGICAL_AND() { - // Init, Free, Prepare, Eval are satisfying the Interface required by - // TfLiteRegistration. - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/LogicalAndEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, nullptr, LogicalAndEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/logistic.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/logistic.cc index 77f94ec0..f8ac1c23 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/logistic.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/logistic.cc @@ -106,13 +106,6 @@ TfLiteStatus LogisticEval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_LOGISTIC() { - return {/*init=*/LogisticInit, - /*free=*/nullptr, - /*prepare=*/LogisticPrepare, - /*invoke=*/LogisticEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(LogisticInit, LogisticPrepare, LogisticEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_eval.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_eval.cc new file mode 100644 index 00000000..f157a8d0 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_eval.cc @@ -0,0 +1,2955 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/micro/kernels/lstm_eval.h" + +#include +#include +#include +#include + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/micro_tensor_utils.h" +namespace tflite { +namespace { + +void ComputeRowSums( + int32_t* input_to_input_row_sums, int32_t* input_to_forget_row_sums, + int32_t* input_to_cell_row_sums, int32_t* input_to_output_row_sums, + int32_t* aux_input_to_input_row_sums, int32_t* aux_input_to_forget_row_sums, + int32_t* aux_input_to_cell_row_sums, int32_t* aux_input_to_output_row_sums, + int32_t* recurrent_to_input_row_sums, int32_t* recurrent_to_forget_row_sums, + int32_t* recurrent_to_cell_row_sums, int32_t* recurrent_to_output_row_sums, + int32_t* projection_weights_row_sums, int32_t* row_sums, int n_cell, + int n_input, int n_aux_input, int n_output, + const int8_t* input_to_input_weights_ptr, + const int8_t* input_to_forget_weights_ptr, + const int8_t* input_to_cell_weights_ptr, + const int8_t* input_to_output_weights_ptr, + const int8_t* aux_input_to_input_weights_ptr, + const int8_t* aux_input_to_forget_weights_ptr, + const int8_t* aux_input_to_cell_weights_ptr, + const int8_t* aux_input_to_output_weights_ptr, + const int8_t* recurrent_to_input_weights_ptr, + const int8_t* recurrent_to_forget_weights_ptr, + const int8_t* recurrent_to_cell_weights_ptr, + const int8_t* recurrent_to_output_weights_ptr, + const int8_t* projection_weights_ptr, bool use_cifg, + const float* aux_input_ptr) { + // Compute the row sums for dequantization + if (!use_cifg) { + micro_tensor_utils::ReductionSumVector( + input_to_input_weights_ptr, input_to_input_row_sums, n_cell, n_input); + } + micro_tensor_utils::ReductionSumVector( + input_to_forget_weights_ptr, input_to_forget_row_sums, n_cell, n_input); + micro_tensor_utils::ReductionSumVector( + input_to_cell_weights_ptr, input_to_cell_row_sums, n_cell, n_input); + micro_tensor_utils::ReductionSumVector( + input_to_output_weights_ptr, input_to_output_row_sums, n_cell, n_input); + + if (aux_input_ptr) { + if (!use_cifg) { + micro_tensor_utils::ReductionSumVector(aux_input_to_input_weights_ptr, + aux_input_to_input_row_sums, + n_cell, n_aux_input); + } + micro_tensor_utils::ReductionSumVector(aux_input_to_forget_weights_ptr, + aux_input_to_forget_row_sums, n_cell, + n_aux_input); + micro_tensor_utils::ReductionSumVector(aux_input_to_cell_weights_ptr, + aux_input_to_cell_row_sums, n_cell, + n_aux_input); + micro_tensor_utils::ReductionSumVector(aux_input_to_output_weights_ptr, + aux_input_to_output_row_sums, n_cell, + n_aux_input); + } + if (!use_cifg) { + micro_tensor_utils::ReductionSumVector(recurrent_to_input_weights_ptr, + recurrent_to_input_row_sums, n_cell, + n_output); + } + micro_tensor_utils::ReductionSumVector(recurrent_to_forget_weights_ptr, + recurrent_to_forget_row_sums, n_cell, + n_output); + micro_tensor_utils::ReductionSumVector(recurrent_to_cell_weights_ptr, + recurrent_to_cell_row_sums, n_cell, + n_output); + micro_tensor_utils::ReductionSumVector(recurrent_to_output_weights_ptr, + recurrent_to_output_row_sums, n_cell, + n_output); + + if (projection_weights_ptr != nullptr) { + micro_tensor_utils::ReductionSumVector( + projection_weights_ptr, projection_weights_row_sums, n_output, n_cell); + } +} + +// Calculates a single LSTM gate. +// +// Implements the following formula: (* is matrix multiply) +// gate = activate(W_input * input + W_aux * aux_input + +// W_peephole * cell + W_recurrent * prev_output + bias) +// with layer norm: +// gate = activate(W_norm * normalize(...) + bias) // not adding bias inside +// +// Activation is sigmoid except for the "cell" gate (configurable, usually tanh) +// +// Parameters: +// Input vectors (to LSTM): | Size: | Optional? +// input | n_input | +// aux_input | n_aux_input | y (bidir LSTM) +// Input vectors (persistent states): +// output_state | n_output | +// cell_state | n_cell | +// 'Constant' inputs: +// input_to_gate_weights | n_cell * n_input | +// aux_input_to_gate_weights | n_cell * n_aux_input | y (bidir LSTM) +// recurrent_to_gate_weights | n_cell * n_output | +// cell_to_gate_weights | n_cell | y (peephole) +// gate_bias | n_cell | +// layer_norm_coefficients | n_cell | y (layer norm) +// Output vector: +// gate | n_cell | +// Scalar parameters: +// n_batch - batch size / number of vectors +// n_input, n_aux_input, n_output, n_cell - size of vectors. +// activation - activation to use. +// is_input_all_zeros, is_aux_input_all_zeros - if input vectors are all zero. +// use_layer_norm - if doing layer norm LSTM. +inline void CalculateLstmGateFloat( + const float* input, const float* input_to_gate_weights, + const float* aux_input, const float* aux_input_to_gate_weights, + const float* output_state, const float* recurrent_to_gate_weights, + const float* cell_state, const float* cell_to_gate_weights, + const float* layer_norm_coefficients, const float* gate_bias, + const int n_batch, const int n_input, const int n_aux_input, + const int n_output, const int n_cell, + const TfLiteFusedActivation activation, float* gate, + const bool is_input_all_zeros, const bool is_aux_input_all_zeros) { + const bool use_peephole = (cell_to_gate_weights != nullptr); + const bool use_layer_norm = (layer_norm_coefficients != nullptr); + + // Initialize scratch buffers with bias for regular lstm or initialize with + // zero for layer norm lstm. + if (use_layer_norm) { + memset(gate, 0, n_cell * n_batch * sizeof(float)); + } else { + micro_tensor_utils::VectorBatchVectorAssign(gate_bias, n_cell, n_batch, + gate); + } + // For each batch and cell: compute input_weight * input. + // Skip if input is all zeros. + if (!is_input_all_zeros) { + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + input_to_gate_weights, n_cell, n_input, input, n_batch, gate); + } + // For each batch and cell: compute aux_input_weight * aux_input. + // Skip if auxiliary input is not available or all zeros. + if (!is_aux_input_all_zeros) { + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + aux_input_to_gate_weights, n_cell, n_aux_input, aux_input, n_batch, + gate); + } + // For each batch and cell: compute recurrent_weight * output_state. + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + recurrent_to_gate_weights, n_cell, n_output, output_state, n_batch, gate); + // For each batch and cell: compute cell_weight .* cell_state (peephole LSTM) + if (use_peephole) { + micro_tensor_utils::VectorBatchVectorCwiseProductAccumulate( + cell_to_gate_weights, n_cell, cell_state, n_batch, gate); + } + // Do layer normalization (if layer norm LSTM) + if (use_layer_norm) { + micro_tensor_utils::MeanStddevNormalization(gate, gate, n_cell, n_batch); + micro_tensor_utils::VectorBatchVectorCwiseProduct( + layer_norm_coefficients, n_cell, gate, n_batch, gate); + micro_tensor_utils::VectorBatchVectorAdd(gate_bias, n_cell, n_batch, gate); + } + // Apply activation + micro_tensor_utils::ApplyActivationToVector(gate, n_batch * n_cell, + activation, gate); +} + +// Updates the LSTM cell state, used by both float and hybrid LSTM versions. +// +// Implements the following formula: +// cell_state_new = clip(forget_gate * cell_state + input_gate * cell_gate) +// +// With CIFG LSTM, input gate is replaced by (1-forget_gate). +// +// Parameters: +// - n_batch, n_cell: sizes of vectors +// - cell_state: input/output vector, size n_batch*n_cell +// - input_gate: input vector, size n_batch*n_cell. +// - forget_gate: input/scratch vector, size n_batch*n_cell, modified with CIFG +// - cell_gate: input vector, size n_batch*n_cell. +// - use_cifg: use 1-forget_gate instead of input_gate. +// - clip: if > 0, clip the resulting cell state to [-clip, +clip]. +void UpdateLstmCellFloat(int n_batch, int n_cell, float* cell_state, + const float* input_gate, float* forget_gate, + const float* cell_gate, bool use_cifg, float clip) { + micro_tensor_utils::VectorVectorCwiseProduct(forget_gate, cell_state, + n_batch * n_cell, cell_state); + + if (use_cifg) { + // With CIFG, input_gate = 1-forget_gate. Use the forget_gate array as + // scratch, as input_gate array is not allocated in this case. (Be careful + // not to write to the scratch before reading the forget gate data.) + float* scratch = forget_gate; + micro_tensor_utils::Sub1Vector(forget_gate, n_batch * n_cell, scratch); + micro_tensor_utils::VectorVectorCwiseProductAccumulate( + cell_gate, scratch, n_batch * n_cell, cell_state); + } else { + micro_tensor_utils::VectorVectorCwiseProductAccumulate( + cell_gate, input_gate, n_batch * n_cell, cell_state); + } + if (clip > 0.0f) { + micro_tensor_utils::CwiseClipping(cell_state, n_batch * n_cell, clip); + } +} + +// Calculates the output state tensor of an LSTM step. +// +// Implements the following formula: +// output_no_projection = output_gate .* activate(cell_state) +// (elementwise vector product) +// If no projection is used: +// output = output_state = output_no_projection +// With projection: +// output = output_state = clip(W*output_no_projection + bias) +// +// Output might not have a different 'stride' than n_batch, so we need to copy. +// +// Parameters: +// - n_batch: batches: the number of distinct vectors in each array. +// - n_cell, n_output: sizes of vectors. +// - cell_state, output_gate: input vectors, size n_batch*n_cell. +// - projection_weights, projection_weights_scale, projection_bias: +// constant inputs, describing projection matrix and bias. +// - proj_clip: if > 0, clip the output of the projection. +// - output_state: output vector, size n_batch*n_output. Must be contigous. +// - scratch: scratch area, size n_batch*n_cell. +void CalculateLstmOutputFloat(int n_batch, int n_cell, int n_output, + const float* cell_state, const float* output_gate, + TfLiteFusedActivation activation, + const float* projection_weights, + const float* projection_bias, + const float proj_clip, float* output_state, + float* scratch) { + micro_tensor_utils::ApplyActivationToVector(cell_state, n_batch * n_cell, + activation, scratch); + micro_tensor_utils::VectorVectorCwiseProduct(output_gate, scratch, + n_batch * n_cell, scratch); + + const bool use_projection = (projection_weights != nullptr); + const bool use_projection_bias = (projection_bias != nullptr); + + if (use_projection) { + if (use_projection_bias) { + micro_tensor_utils::VectorBatchVectorAssign(projection_bias, n_output, + n_batch, output_state); + } else { + memset(output_state, 0, n_batch * n_output * sizeof(float)); + } + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + projection_weights, n_output, n_cell, scratch, n_batch, output_state); + if (proj_clip > 0.0f) { + micro_tensor_utils::CwiseClipping(output_state, n_batch * n_output, + proj_clip); + } + } else { + std::memcpy(output_state, scratch, n_batch * n_output * sizeof(float)); + } +} + +// Calculates a single LSTM gate, hybrid version. +// Implements the same functionality as CalculateLstmGateFloat. +void CalculateLstmGateHybrid( + // Input and weights + const int8_t* input, const float* input_sf, const int32_t* input_zp, + const int8_t* input_to_gate_weights, + const uint8_t* input_to_gate_weights_ledger, + const float input_to_gate_weights_scale, int32_t* input_to_gate_row_sums, + // Aux input and weights + const int8_t* aux_input, const float* aux_input_sf, + const int32_t* aux_input_zp, const int8_t* aux_input_to_gate_weights, + const float aux_input_to_gate_weights_scale, + int32_t* aux_input_to_gate_row_sums, + // Output state and weights + const int8_t* output_state, const float* output_state_sf, + const int32_t* output_state_zp, const int8_t* recurrent_to_gate_weights, + const uint8_t* recurrent_to_gate_weights_ledger, + const float recurrent_to_gate_weights_scale, + int32_t* recurrent_to_gate_row_sums, + // Cell state and weights (peephole LSTM) + const float* cell_state, const int8_t* cell_to_gate_weights, + const float cell_to_gate_weights_scale, + // Layer normalization coefficients (layer norm LSTM) + gate bias + const float* layer_norm_coefficients, const float* gate_bias, + // Array sizes + const int n_batch, const int n_input, const int n_aux_input, + const int n_output, const int n_cell, + const TfLiteFusedActivation activation, + // Output + float* gate, + // Parameters for performance optimizations + const bool is_input_all_zeros, const bool is_aux_input_all_zeros, + const bool is_output_state_all_zeros, bool* compute_row_sums, + // Scratch arrays + float* scratch0, // size: n_batch + float* scratch1, // size: n_cell, only used if peephole LSTM + float* scales, // size: n_batch + int32_t* accum_scratch // For MatrixBatchVectorMultiplyAccumulate +) { + const bool use_peephole = (cell_to_gate_weights != nullptr); + const bool use_layer_norm = (layer_norm_coefficients != nullptr); + + // Initialize scratch buffers with bias for regular lstm or initialize with + // zero for layer norm lstm. + if (use_layer_norm) { + memset(gate, 0, n_cell * n_batch * sizeof(float)); + } else { + micro_tensor_utils::VectorBatchVectorAssign(gate_bias, n_cell, n_batch, + gate); + } + // For each batch and cell: compute input_weight * input. + // Skip if input is all zeros. + if (!is_input_all_zeros) { + if (input_to_gate_weights_ledger != nullptr) { + for (int i = 0; i < n_batch; i++) { + scales[i] = input_to_gate_weights_scale * input_sf[i]; + } + micro_tensor_utils::SparseMatrixBatchVectorMultiplyAccumulate( + input_to_gate_weights, input_to_gate_weights_ledger, n_cell, n_input, + input, scales, n_batch, gate); + + } else { + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + input_to_gate_weights, n_cell, n_input, input, + input_to_gate_weights_scale, input_sf, n_batch, gate, + /*per_channel_scale=*/nullptr, input_zp, accum_scratch, + input_to_gate_row_sums, compute_row_sums, scratch0, nullptr); + } + } + // For each batch and cell: compute aux_input_weight * aux_input. + // Skip if auxiliary input is not available or all zeros. + if (!is_aux_input_all_zeros) { + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + aux_input_to_gate_weights, n_cell, n_aux_input, aux_input, + aux_input_to_gate_weights_scale, aux_input_sf, n_batch, gate, + /*per_channel_scale=*/nullptr, aux_input_zp, accum_scratch, + aux_input_to_gate_row_sums, compute_row_sums, scratch0, nullptr); + } + // For each batch and cell: compute recurrent_weight * output_state. + // Skip if output state is all zeros. + if (!is_output_state_all_zeros) { + if (recurrent_to_gate_weights_ledger != nullptr) { + for (int i = 0; i < n_batch; i++) { + scales[i] = recurrent_to_gate_weights_scale * input_sf[i]; + } + micro_tensor_utils::SparseMatrixBatchVectorMultiplyAccumulate( + recurrent_to_gate_weights, recurrent_to_gate_weights_ledger, n_cell, + n_output, output_state, scales, n_batch, gate); + } else { + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + recurrent_to_gate_weights, n_cell, n_output, output_state, + recurrent_to_gate_weights_scale, output_state_sf, n_batch, gate, + /*per_channel_scale=*/nullptr, output_state_zp, accum_scratch, + recurrent_to_gate_row_sums, compute_row_sums, scratch0, nullptr); + } + } + // For each batch and cell: compute cell_weight .* cell_state (peephole LSTM) + if (use_peephole) { + float* recovered_cell_weights = scratch1; + micro_tensor_utils::VectorScalarMultiply(cell_to_gate_weights, n_cell, + cell_to_gate_weights_scale, + recovered_cell_weights); + micro_tensor_utils::VectorBatchVectorCwiseProductAccumulate( + recovered_cell_weights, n_cell, cell_state, n_batch, gate); + } + // Do layer normalization (if layer norm LSTM) + if (use_layer_norm) { + micro_tensor_utils::MeanStddevNormalization(gate, gate, n_cell, n_batch); + micro_tensor_utils::VectorBatchVectorCwiseProduct( + layer_norm_coefficients, n_cell, gate, n_batch, gate); + micro_tensor_utils::VectorBatchVectorAdd(gate_bias, n_cell, n_batch, gate); + } + // Apply activation + micro_tensor_utils::ApplyActivationToVector(gate, n_cell * n_batch, + activation, gate); +} + +// Calculates the output state tensor of an LSTM step. See Float version too. +// +// Parameters: +// - n_batch: batches: the number of distinct vectors in each array. +// - n_cell, n_output: sizes of vectors. +// - cell_state, output_gate: input vectors, size n_batch*n_cell. +// - projection_weights, projection_weights_scale, projection_bias: +// constant inputs, describing projection matrix and bias. +// - proj_clip: if > 0, clip the output of the projection. +// - output_state: output vector, size n_batch*n_output. Must be contigous. +// - asymmetric_quantize_inputs: parameter to control quantization. +// - projection_weights_row_sums, compute_row_sums: Data for optimized +// MatrixBatchVectorMultiplyAccumulate. +// - scratch0: scratch area of size n_batch*n_cell +// - scratch1: scratch area of size n_batch*n_cell +// - scratch2: scratch area of size n_batch +// - scratch3: scratch area of size n_batch +// - scratch4: scratch area used by MatrixBatchVectorMultiplyAccumulate +// - scales: scratch area of size n_batch +void CalculateLstmOutputHybrid( + int n_batch, int n_cell, int n_output, const float* cell_state, + const float* output_gate, TfLiteFusedActivation activation, + const int8_t* projection_weights, const uint8_t* projection_weights_ledger, + float projection_weights_scale, const float* projection_bias, + const float proj_clip, float* output_state, bool asymmetric_quantize_inputs, + int32_t* projection_weights_row_sums, bool* compute_row_sums, + float* scratch0, int8_t* scratch1, float* scratch2, int32_t* scratch3, + int32_t* scratch4, float* scales) { + micro_tensor_utils::ApplyActivationToVector(cell_state, n_batch * n_cell, + activation, scratch0); + micro_tensor_utils::VectorVectorCwiseProduct(output_gate, scratch0, + n_batch * n_cell, scratch0); + + const bool use_projection = (projection_weights != nullptr); + const bool use_projection_bias = (projection_bias != nullptr); + + if (use_projection) { + if (use_projection_bias) { + micro_tensor_utils::VectorBatchVectorAssign(projection_bias, n_output, + n_batch, output_state); + } else { + memset(output_state, 0, n_batch * n_output * sizeof(float)); + } + if (!micro_tensor_utils::IsZeroVector(scratch0, n_batch * n_cell)) { + // Save quantization and matmul computation for all zero output. + micro_tensor_utils::BatchQuantizeFloats(scratch0, n_batch, n_cell, + scratch1, scratch2, scratch3, + asymmetric_quantize_inputs); + if (projection_weights_ledger != nullptr) { + for (int i = 0; i < n_batch; i++) { + scales[i] = projection_weights_scale * scratch2[i]; + } + micro_tensor_utils::SparseMatrixBatchVectorMultiplyAccumulate( + projection_weights, projection_weights_ledger, n_output, n_cell, + scratch1, scales, n_batch, output_state); + } else { + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + projection_weights, n_output, n_cell, scratch1, + projection_weights_scale, scratch2, n_batch, output_state, + /*per_channel_scale=*/nullptr, scratch3, scratch4, + projection_weights_row_sums, compute_row_sums, scratch2, nullptr); + } + } + if (proj_clip > 0.0f) { + micro_tensor_utils::CwiseClipping(output_state, n_batch * n_output, + proj_clip); + } + } else { + std::memcpy(output_state, scratch0, n_batch * n_output * sizeof(float)); + } +} + +// Calculates a single LSTM gate, int8x8_16 version. +// Implements the same functionality as CalculateLstmGateFloat. +void CalculateLstmGateInteger8x8_16( + // Input and weights + const int8_t* input, const int8_t* input_to_gate_weights, + const int32_t* input_to_gate_bias, const int32_t input_to_gate_scale_a, + const int32_t input_to_gate_scale_b, + // Output state and weights + const int8_t* output_state, const int8_t* recurrent_to_gate_weights, + const int32_t* recurrent_to_gate_bias, + const int32_t recurrent_to_gate_scale_a, + const int32_t recurrent_to_gate_scale_b, + // Cell state and weights + const int16_t* cell_state, const int16_t* cell_to_gate_weights, + const int32_t cell_to_gate_scale_a, const int32_t cell_to_gate_scale_b, + // Layer normalization parameters (layer norm LSTM) + const int16_t* layer_norm_coefficients, const int32_t* layer_norm_bias, + const int32_t layer_norm_input_scale_a, + const int32_t layer_norm_input_scale_b, + const int32_t layer_norm_variance_guard, + // Array sizes + const int n_batch, const int n_input, const int n_output, const int n_cell, + const TfLiteFusedActivation activation, + // Output + int16_t* gate, + // Parameters for performance optimizations + // Scratch arrays + int32_t* scratch5) { + const bool use_peephole = (cell_to_gate_weights != nullptr); + const bool use_layer_norm = (layer_norm_coefficients != nullptr); + + // Initialize scratch buffers with zeros. Note that unlike float and hybrid + // versions, bias is only used in layer normalization. + memset(gate, 0, n_batch * n_cell * sizeof(int16_t)); + // For each batch and cell: compute input_weight * input. + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + input, input_to_gate_bias, input_to_gate_weights, input_to_gate_scale_a, + input_to_gate_scale_b, n_batch, n_input, n_cell, 0, scratch5, gate, + nullptr); + // Note: no aux_input. + + // For each batch and cell: compute recurrent_weight * output_state. + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + output_state, recurrent_to_gate_bias, recurrent_to_gate_weights, + recurrent_to_gate_scale_a, recurrent_to_gate_scale_b, n_batch, n_output, + n_cell, 0, scratch5, gate, nullptr); + // For each batch and cell: compute cell_weight * cell_state (peephole LSTM) + if (use_peephole) { + micro_tensor_utils::VectorBatchVectorCwiseProductAccumulate( + cell_to_gate_weights, n_output, cell_state, n_batch, + cell_to_gate_scale_a, cell_to_gate_scale_b, gate); + } + // Do layer normalization (if layer norm LSTM) + if (use_layer_norm) { + micro_tensor_utils::ApplyLayerNorm( + gate, layer_norm_coefficients, layer_norm_bias, + layer_norm_input_scale_a, layer_norm_input_scale_b, + layer_norm_variance_guard, n_batch, n_cell, gate); + } + // Apply activation + switch (activation) { + case kTfLiteActSigmoid: + micro_tensor_utils::ApplySigmoid(gate, n_batch, n_cell, gate); + break; + case kTfLiteActTanh: + micro_tensor_utils::ApplyTanh(3, gate, n_batch, n_cell, gate); + break; + default: + // Only Sigmoid or Tanh is used. + TFLITE_ASSERT_FALSE; + } +} + +// Updates the LSTM cell state, used by both integer LSTM versions. +// Also see UpdateLstmCellFloat. +// +// Parameters: +// - n_batch, n_cell: sizes of vectors +// - cell_state: input/output vector, size n_batch*n_cell +// - cell_state_scale: scaling factor of cell state. +// - input_gate: input vector, size n_batch*n_cell. +// - forget_gate: input/scratch vector, size n_batch*n_cell, always modified. +// - cell_gate: input vector, size n_batch*n_cell. +// - use_cifg: use 1-forget_gate instead of input_gate. +// - clip: if > 0, clip the resulting cell state to [-clip, +clip]. +void UpdateLstmCellInteger(int n_batch, int n_cell, int16_t* cell_state, + int32_t cell_state_scale, const int16_t* input_gate, + int16_t* forget_gate, const int16_t* cell_gate, + bool use_cifg, int16_t clip) { + // Use the forget_gate array as scratch, as input_gate array is not allocated + // in CIFG case. (Be careful not to write to the scratch before reading the + // forget gate data.) + int16_t* scratch = forget_gate; + + micro_tensor_utils::CwiseMul(forget_gate, cell_state, n_batch, n_cell, 15, + cell_state); + if (use_cifg) { + micro_tensor_utils::Sub1Vector(forget_gate, n_batch * n_cell, scratch); + micro_tensor_utils::CwiseMul(scratch, cell_gate, n_batch, n_cell, + 30 + cell_state_scale, scratch); + } else { + micro_tensor_utils::CwiseMul(input_gate, cell_gate, n_batch, n_cell, + 30 + cell_state_scale, scratch); + } + micro_tensor_utils::CwiseAdd(cell_state, scratch, n_batch, n_cell, + cell_state); + + if (clip > 0) { + micro_tensor_utils::CwiseClipping(cell_state, n_batch * n_cell, clip); + } +} + +// Calculates the output state tensor of an LSTM step. See Float and hybrid +// versions as well. +// +// Parameters: +// - n_batch: batches: the number of distinct vectors in each array. +// - n_cell, n_output: sizes of vectors. +// - cell_state, output_gate: input vectors, size n_batch*n_cell. +// - cell_state_scale: scaling of cell_state. +// - hidden_scale_[a|b]: effective scale of cell_state.*output_gate +// - hidden_zp: zero_point for cell_state.*output_gate +// - projection_weights, proj_scale_[a|b], projection_bias: +// constant inputs, describing projection matrix and bias. +// - output_state_zp: zero point of output_state. (Input, calibrated value.) +// - quantized_proj_clip: if > 0, clip the output of the projection. +// - output_state: output vector, size n_batch*n_output. Must be contigous. +// - scratch0: scratch area of size n_batch*n_cell +// - scratch1: scratch area of size n_batch*n_cell +// - scratch2: scratch area used by MatrixBatchVectorMultiplyAccumulate +void CalculateLstmOutputInteger8x8_16( + int n_batch, int n_cell, int n_output, const int16_t* cell_state, + int32_t cell_state_scale, const int16_t* output_gate, + int32_t hidden_scale_a, int32_t hidden_scale_b, int32_t hidden_zp, + const int8_t* projection_weights, int32_t proj_scale_a, + int32_t proj_scale_b, const int32_t* projection_bias, + int32_t output_state_zp, int8_t quantized_proj_clip, int8_t* output_state, + int16_t* scratch0, int8_t* scratch1, int32_t* scratch2) { + // Note: unlike float/hybrid, the activation is always Tanh. + micro_tensor_utils::ApplyTanh(15 + cell_state_scale, cell_state, n_batch, + n_cell, scratch0); + micro_tensor_utils::CwiseMul(output_gate, scratch0, hidden_scale_a, + hidden_scale_b, n_batch, n_cell, hidden_zp, + scratch1); + + const bool use_projection = (projection_weights != nullptr); + + if (use_projection) { + // Note: no bias like in float/hybrid + memset(output_state, 0, n_batch * n_output * sizeof(int8_t)); + micro_tensor_utils::MatrixBatchVectorMultiplyAccumulate( + scratch1, projection_bias, projection_weights, proj_scale_a, + proj_scale_b, n_batch, n_cell, n_output, output_state_zp, scratch2, + output_state, nullptr); + if (quantized_proj_clip > 0) { + micro_tensor_utils::CwiseClipping(output_state, n_batch * n_output, + quantized_proj_clip); + } + } else { + std::memcpy(output_state, scratch1, n_batch * n_output * sizeof(int8_t)); + } +} + +// Calculates a single LSTM gate, int8x8_8 version. +// Implements the same functionality as CalculateLstmGateFloat. +void CalculateLstmGateInteger8x8_8( + // Inputs and weights + const int8_t* input, int32_t input_zp, const int8_t* input_to_gate_weight, + const int32_t input_to_gate_scale_a, const int32_t input_to_gate_scale_b, + const int32_t input_times_weights_scale_a, + const int32_t input_times_weights_scale_b, + const int32_t input_times_weights_zp, + // Output state and weights + const int8_t* output_state, const int32_t output_state_zp, + const int8_t* recurrent_to_gate_weight, + const int32_t recurrent_to_gate_scale_a, + const int32_t recurrent_to_gate_scale_b, + const int32_t output_state_times_weights_scale_a, + const int32_t output_state_times_weights_scale_b, + const int32_t output_state_times_weights_zp, + // Layer normalization parameters (layer norm LSTM) + const int16_t* layer_norm_gate_weight, + const int32_t layer_norm_gate_scale_a, + const int32_t layer_norm_gate_scale_b, const int32_t* gate_bias, + // Array sizes + const int n_batch, const int n_input, const int n_output, const int n_cell, + const TfLiteFusedActivation activation, + // Output + int16_t* gate, + // Scratch arrays, both sized n_batch*n_cell + int8_t* scratch0, int8_t* scratch1) { + // Multiply input * input_weights => scratch0 + micro_tensor_utils::MatrixBatchVectorMultiply( + input, input_zp, input_to_gate_weight, input_to_gate_scale_a, + input_to_gate_scale_b, n_batch, n_input, n_cell, scratch0, + input_times_weights_zp); + // Multiply output_state * recurrent_weights => scratch1 + micro_tensor_utils::MatrixBatchVectorMultiply( + output_state, output_state_zp, recurrent_to_gate_weight, + recurrent_to_gate_scale_a, recurrent_to_gate_scale_b, n_batch, n_output, + n_cell, scratch1, output_state_times_weights_zp); + // Add scratch0 + scratch1 => gate + micro_tensor_utils::TwoGateSaturatingAdd( + scratch0, input_times_weights_zp, scratch1, output_state_times_weights_zp, + input_times_weights_scale_a, input_times_weights_scale_b, + output_state_times_weights_scale_a, output_state_times_weights_scale_b, + n_batch, n_cell, gate); + // Apply layer normalization. + micro_tensor_utils::ApplyLayerNormFloat( + gate, layer_norm_gate_weight, layer_norm_gate_scale_a, + layer_norm_gate_scale_b, gate_bias, n_batch, n_cell, gate); + // Apply activation. + switch (activation) { + case kTfLiteActSigmoid: + micro_tensor_utils::ApplySigmoidFloat(gate, n_batch, n_cell, gate); + break; + case kTfLiteActTanh: + micro_tensor_utils::ApplyTanhFloat(gate, n_batch, n_cell, -12, gate); + break; + default: + // Only Sigmoid or Tanh is used. + TFLITE_ASSERT_FALSE; + } +} + +// Calculates the output state tensor of an LSTM step. See Float and hybrid +// versions as well. +// +// Parameters: +// - n_batch: batches: the number of distinct vectors in each array. +// - n_cell, n_output: sizes of vectors. +// - cell_state, output_gate: input vectors, size n_batch*n_cell. +// - projection_weights, proj_scale_[a|b], projection_bias: +// constant inputs, describing projection matrix and bias. +// - output_state_zp: zero point of the output state. +// - quantized_proj_clip: if > 0, clip the output of the projection. +// - output_state: output vector, size n_batch*n_output. Must be contigous. +// - scratch: scratch area of size n_batch*n_cell +void CalculateLstmOutputInteger8x8_8( + int n_batch, int n_cell, int n_output, const int16_t* cell_state, + const int16_t* output_gate, const int8_t* projection_weights, + int32_t proj_scale_a, int32_t proj_scale_b, const int32_t* projection_bias, + int32_t output_state_zp, int32_t quantized_proj_clip, int8_t* output_state, + int16_t* scratch) { + // Note: unlike float/hybrid, the activation is always Tanh. + micro_tensor_utils::ApplyTanhFloat(cell_state, n_batch, n_cell, -15, scratch); + micro_tensor_utils::CwiseMul(output_gate, scratch, n_batch, n_cell, + 15 + 15 - 15, scratch); + // Note: no bias like in float/hybrid + micro_tensor_utils::MatrixBatchVectorMultiply( + scratch, projection_weights, proj_scale_a, proj_scale_b, projection_bias, + n_batch, n_cell, n_output, output_state_zp, output_state); + if (quantized_proj_clip > 0) { + micro_tensor_utils::CwiseClipping(output_state, n_batch * n_output, + quantized_proj_clip); + } +} + +// Performs an LSTM batch inference step for input specified by input_ptr. +// The LSTM cell is specified by the pointers to its weights (*_weights_ptr) and +// biases (*_bias_ptr), and buffers (*_scratch), along with additional +// parameters: +// - params: various LSTM params including activation, clipping, etc., +// - n_batch: size of batch, +// - n_cell: number of cells (or units), +// - n_input: the input size, +// - n_aux_input: the auxiliary input size. +// - n_output: the output size. +// - output_batch_leading_dim: the leading dimension of the output buffer. +// +// Input of size 'n_batch * n_input': +// input_ptr +// Input of size 'n_batch * n_aux_input': +// aux_input_ptr - optional (can be nullptr) +// +// LSTM weights: +// Input weights of size 'n_cell * n_input': +// input_to_input_weights - optional +// input_to_forget_weights +// input_to_cell_weights +// input_to_output_weights +// Auxiliary input weights of size 'n_cell * n_aux_input': +// aux_input_to_input_weights - optional +// aux_input_to_forget_weights - optional +// aux_input_to_cell_weights - optional +// aux_input_to_output_weights - optional +// Recurrent weights of size 'n_cell * n_output': +// recurrent_to_input_weights - optional +// recurrent_to_forget_weights +// recurrent_to_cell_weights +// recurrent_to_input_weights +// Peephole weights of size 'n_cell', representing diagonal matrices. +// cell_to_input_weights - optional +// cell_to_cell_weights - optional +// cell_to_output_weights - optional +// Projection weights of size 'n_output * n_cell' +// projection_weights_ptr - optional +// Gate biases of size 'n_cell': +// input_gate_bias_ptr - optional +// forget_gate_bias_ptr +// cell_gate_bias_ptr +// output_gate_bias_ptr +// +// Layer norm coefficients of size 'n_cell', representing diagonal matrices. +// input_layer_norm_coefficients_ptr - optional +// forget_layer_norm_coefficients_ptr - optional +// cell_layer_norm_coefficients_ptr - optional +// output_layer_norm_coefficients_ptr - optional +// +// The pointers to the cell and output state and the output are updated. +// +// The pointers input_ptr, aux_input_ptr, and output_ptr point to data aligned +// in batch_major order, and each step processes batch_size many inputs from +// input_ptr, and updates batch_size many cell and output states. +// +// The output_batch_dim is output.shape[-1], i.e. the outermost dimension of the +// output tensor, and in most cases will be equal to n_output. It is usually not +// when we want to store the LSTM output into a slice of the output tensor, e.g. +// for bidirectional LSTMs with merge_outputs. In this case, the batched +// operations cannot be used since they assume that the batched outputs are +// contiguous, and we manually loop over the batched outputs. +inline void LstmStepFloat( + const float* input_ptr, const float* input_to_input_weights_ptr, + const float* input_to_forget_weights_ptr, + const float* input_to_cell_weights_ptr, + const float* input_to_output_weights_ptr, const float* aux_input_ptr, + const float* aux_input_to_input_weights_ptr, + const float* aux_input_to_forget_weights_ptr, + const float* aux_input_to_cell_weights_ptr, + const float* aux_input_to_output_weights_ptr, + const float* recurrent_to_input_weights_ptr, + const float* recurrent_to_forget_weights_ptr, + const float* recurrent_to_cell_weights_ptr, + const float* recurrent_to_output_weights_ptr, + const float* cell_to_input_weights_ptr, + const float* cell_to_forget_weights_ptr, + const float* cell_to_output_weights_ptr, + const float* input_layer_norm_coefficients_ptr, + const float* forget_layer_norm_coefficients_ptr, + const float* cell_layer_norm_coefficients_ptr, + const float* output_layer_norm_coefficients_ptr, + const float* input_gate_bias_ptr, const float* forget_gate_bias_ptr, + const float* cell_gate_bias_ptr, const float* output_gate_bias_ptr, + const float* projection_weights_ptr, const float* projection_bias_ptr, + const TfLiteLSTMParams* params, int n_batch, int n_cell, int n_input, + int n_aux_input, int n_output, int output_batch_leading_dim, + float* output_state_ptr, float* cell_state_ptr, float* scratch0, + float* scratch1, float* scratch2, float* scratch3, float* output_ptr) { + // Since we have already checked that weights are all there or none, we can + // check the existence of only one to the get the condition. + const bool use_cifg = (input_to_input_weights_ptr == nullptr); + + // Make named scratch buffers. + float* input_gate_scratch = scratch0; + float* forget_gate_scratch = scratch1; + float* cell_gate_scratch = scratch2; + float* output_gate_scratch = scratch3; + + // Check if inputs are all zeros so we can skip some computations. + const bool is_input_all_zeros = + micro_tensor_utils::IsZeroVector(input_ptr, n_batch * n_input); + const bool is_aux_input_all_zeros = + (aux_input_ptr == nullptr || + micro_tensor_utils::IsZeroVector(aux_input_ptr, n_batch * n_aux_input)); + if (!use_cifg) { + // Calculate the input gate. (If not CIFG.) + CalculateLstmGateFloat( + input_ptr, input_to_input_weights_ptr, aux_input_ptr, + aux_input_to_input_weights_ptr, output_state_ptr, + recurrent_to_input_weights_ptr, cell_state_ptr, + cell_to_input_weights_ptr, input_layer_norm_coefficients_ptr, + input_gate_bias_ptr, n_batch, n_input, n_aux_input, n_output, n_cell, + /*activation=*/kTfLiteActSigmoid, input_gate_scratch, + is_input_all_zeros, is_aux_input_all_zeros); + } + // Calculate the forget gate. + CalculateLstmGateFloat( + input_ptr, input_to_forget_weights_ptr, aux_input_ptr, + aux_input_to_forget_weights_ptr, output_state_ptr, + recurrent_to_forget_weights_ptr, cell_state_ptr, + cell_to_forget_weights_ptr, forget_layer_norm_coefficients_ptr, + forget_gate_bias_ptr, n_batch, n_input, n_aux_input, n_output, n_cell, + /*activation=*/kTfLiteActSigmoid, forget_gate_scratch, is_input_all_zeros, + is_aux_input_all_zeros); + // Calculate the cell update gate. + CalculateLstmGateFloat(input_ptr, input_to_cell_weights_ptr, aux_input_ptr, + aux_input_to_cell_weights_ptr, output_state_ptr, + recurrent_to_cell_weights_ptr, /*cell_state=*/nullptr, + /*cell_to_gate_weights=*/nullptr, + cell_layer_norm_coefficients_ptr, cell_gate_bias_ptr, + n_batch, n_input, n_aux_input, n_output, n_cell, + params->activation, cell_gate_scratch, + is_input_all_zeros, is_aux_input_all_zeros); + // Update the cell state. + UpdateLstmCellFloat(n_batch, n_cell, cell_state_ptr, input_gate_scratch, + forget_gate_scratch, cell_gate_scratch, use_cifg, + params->cell_clip); + // Calculate output gate. + CalculateLstmGateFloat( + input_ptr, input_to_output_weights_ptr, aux_input_ptr, + aux_input_to_output_weights_ptr, output_state_ptr, + recurrent_to_output_weights_ptr, cell_state_ptr, + cell_to_output_weights_ptr, output_layer_norm_coefficients_ptr, + output_gate_bias_ptr, n_batch, n_input, n_aux_input, n_output, n_cell, + /*activation=*/kTfLiteActSigmoid, output_gate_scratch, is_input_all_zeros, + is_aux_input_all_zeros); + // Update the output state. + CalculateLstmOutputFloat(n_batch, n_cell, n_output, cell_state_ptr, + output_gate_scratch, params->activation, + projection_weights_ptr, projection_bias_ptr, + params->proj_clip, output_state_ptr, scratch2); + // Copy output state to the output. Note that the output's rows may not be + // contiguous (output_batch_leading_dim != n_output). + for (int b = 0; b < n_batch; b++) { + std::memcpy(output_ptr + b * output_batch_leading_dim, + output_state_ptr + b * n_output, n_output * sizeof(float)); + } +} + +// Same as above but with quantized weight matrices. In detail: +// Input of size 'n_batch * n_input': +// input_ptr +// Input of size 'n_batch * n_aux_input': +// aux_input_ptr - optional (can be nullptr) +// +// LSTM weights: +// Quantized input weights of size 'n_cell * n_input': +// input_to_input_weights - optional +// input_to_forget_weights +// input_to_cell_weights +// input_to_input_weights +// Quantized auxiliary input weights of size 'n_cell * n_aux_input': +// aux_input_to_input_weights - optional +// aux_input_to_forget_weights - optional +// aux_input_to_cell_weights - optional +// aux_input_to_output_weights - optional +// Quantized recurrent weights of size 'n_cell * n_output': +// recurrent_to_input_weights - optional +// recurrent_to_forget_weights +// recurrent_to_cell_weights +// recurrent_to_input_weights +// Quantized peephole weights of size 'n_cell', representing diagonal matrices. +// cell_to_input_weights - optional +// cell_to_cell_weights - optional +// cell_to_output_weights - optional +// Quantized projection weights of size 'n_output * n_cell' +// projection_weights_ptr - optional +// Weight scales (scalars) for each of the weights above. +// input_to_input_weights_scale - optional +// input_to_forget_weights_scale +// input_to_cell_weights_scale +// input_to_output_weights_scale +// aux_input_to_input_weights_scale - optional +// aux_input_to_forget_weights_scale - optional +// aux_input_to_cell_weights_scale - optional +// aux_input_to_output_weights_scale - optional +// recurrent_to_input_weights_scale - optional +// recurrent_to_forget_weights_scale +// recurrent_to_cell_weights_scale +// recurrent_to_output_weights_scale +// cell_to_input_weights_scale, +// cell_to_forget_weights_scale, +// cell_to_output_weights_scale, +// projection_weights_scale - optional +// Gate biases of size 'n_cell': +// input_gate_bias_ptr - optional +// forget_gate_bias_ptr +// cell_gate_bias_ptr +// output_gate_bias_ptr +// +// Layer norm coefficients of size 'n_cell', representing diagonal matrices. +// input_layer_norm_coefficients_ptr - optional +// forget_layer_norm_coefficients_ptr - optional +// cell_layer_norm_coefficients_ptr - optional +// output_layer_norm_coefficients_ptr - optional +// +// Temporary pre-allocated storage for quantized values: +// quantized_input_ptr (same size as input_ptr) +// quantized_output_state_ptr (same size as output_state_ptr) +// quantized_output_scratch (same size as cell_state_ptr) +// Temporary pre-allocated storage for recovered values: +// recovered_cell_weights (same size as cell_to_*_weights) +// +// Outputs: +// output_state_ptr - size 'n_batch * n_output' +// cell_state_ptr - size 'n_batch * n_cell' +// output_ptr - size 'n_batch * output_batch_leading_dim' +inline void LstmStepHybrid( + const float* input_ptr, const int8_t* input_to_input_weights_ptr, + const uint8_t* input_to_input_weights_ledger_ptr, + float input_to_input_weights_scale, + const int8_t* input_to_forget_weights_ptr, + const uint8_t* input_to_forget_weights_ledger_ptr, + float input_to_forget_weights_scale, + const int8_t* input_to_cell_weights_ptr, + const uint8_t* input_to_cell_weights_ledger_ptr, + float input_to_cell_weights_scale, + const int8_t* input_to_output_weights_ptr, + const uint8_t* input_to_output_weights_ledger_ptr, + float input_to_output_weights_scale, const float* aux_input_ptr, + const int8_t* aux_input_to_input_weights_ptr, + float aux_input_to_input_weights_scale, + const int8_t* aux_input_to_forget_weights_ptr, + float aux_input_to_forget_weights_scale, + const int8_t* aux_input_to_cell_weights_ptr, + float aux_input_to_cell_weights_scale, + const int8_t* aux_input_to_output_weights_ptr, + float aux_input_to_output_weights_scale, + const int8_t* recurrent_to_input_weights_ptr, + const uint8_t* recurrent_to_input_weights_ledger_ptr, + float recurrent_to_input_weights_scale, + const int8_t* recurrent_to_forget_weights_ptr, + const uint8_t* recurrent_to_forget_weights_ledger_ptr, + float recurrent_to_forget_weights_scale, + const int8_t* recurrent_to_cell_weights_ptr, + const uint8_t* recurrent_to_cell_weights_ledger_ptr, + float recurrent_to_cell_weights_scale, + const int8_t* recurrent_to_output_weights_ptr, + const uint8_t* recurrent_to_output_weights_ledger_ptr, + float recurrent_to_output_weights_scale, + const int8_t* cell_to_input_weights_ptr, float cell_to_input_weights_scale, + const int8_t* cell_to_forget_weights_ptr, + float cell_to_forget_weights_scale, + const int8_t* cell_to_output_weights_ptr, + float cell_to_output_weights_scale, + const float* input_layer_norm_coefficients_ptr, + const float* forget_layer_norm_coefficients_ptr, + const float* cell_layer_norm_coefficients_ptr, + const float* output_layer_norm_coefficients_ptr, + const float* input_gate_bias_ptr, const float* forget_gate_bias_ptr, + const float* cell_gate_bias_ptr, const float* output_gate_bias_ptr, + const int8_t* projection_weights_ptr, + const uint8_t* projection_weights_ledger_ptr, + float projection_weights_scale, const float* projection_bias_ptr, + const TfLiteLSTMParams* params, int n_batch, int n_cell, int n_input, + int n_aux_input, int n_output, int output_batch_leading_dim, + float* scratch0, float* scratch1, float* scratch2, float* scratch3, + float* scales, float* input_sf, float* aux_input_sf, float* output_state_sf, + float* scaling_factors_scratch, float* recovered_cell_weights, + int8_t* quantized_input_ptr, int8_t* quantized_aux_input_ptr, + int8_t* quantized_output_state_ptr, int8_t* quantized_output_scratch, + float* output_state_ptr, float* cell_state_ptr, int32_t* accum_scratch_ptr, + float* output_ptr, int32_t* input_zp, int32_t* aux_input_zp, + int32_t* output_state_zp, int32_t* row_sums, int row_sums_size, + bool* compute_row_sums, bool asymmetric_quantize_inputs) { + // Since we have already checked that weights are all there or none, we + // can check the existence of only one to the get the condition. + const bool use_cifg = (input_to_input_weights_ptr == nullptr); + // Make named scratch buffers for the different gates. + float* input_gate_scratch = scratch0; + float* forget_gate_scratch = scratch1; + float* cell_gate_scratch = scratch2; + float* output_gate_scratch = scratch3; + + int32_t* input_to_input_row_sums = nullptr; + int32_t* input_to_forget_row_sums = nullptr; + int32_t* input_to_cell_row_sums = nullptr; + int32_t* input_to_output_row_sums = nullptr; + int32_t* aux_input_to_input_row_sums = nullptr; + int32_t* aux_input_to_forget_row_sums = nullptr; + int32_t* aux_input_to_cell_row_sums = nullptr; + int32_t* aux_input_to_output_row_sums = nullptr; + int32_t* recurrent_to_input_row_sums = nullptr; + int32_t* recurrent_to_forget_row_sums = nullptr; + int32_t* recurrent_to_cell_row_sums = nullptr; + int32_t* recurrent_to_output_row_sums = nullptr; + int32_t* projection_weights_row_sums = nullptr; + + if (asymmetric_quantize_inputs) { + int num_row_sums = use_cifg ? 6 : 8; + if (aux_input_ptr != nullptr) { + num_row_sums += use_cifg ? 3 : 4; + } + if (projection_weights_ptr != nullptr) { + num_row_sums += ceil(static_cast(n_output) / n_cell); + } + TFLITE_DCHECK(row_sums_size == num_row_sums); + input_to_input_row_sums = row_sums; + input_to_forget_row_sums = + use_cifg ? input_to_input_row_sums : input_to_input_row_sums + n_cell; + input_to_cell_row_sums = input_to_forget_row_sums + n_cell; + input_to_output_row_sums = input_to_cell_row_sums + n_cell; + if (aux_input_ptr != nullptr) { + aux_input_to_input_row_sums = input_to_output_row_sums + n_cell; + aux_input_to_forget_row_sums = use_cifg + ? aux_input_to_input_row_sums + : aux_input_to_input_row_sums + n_cell; + aux_input_to_cell_row_sums = aux_input_to_forget_row_sums + n_cell; + aux_input_to_output_row_sums = aux_input_to_cell_row_sums + n_cell; + } + recurrent_to_input_row_sums = aux_input_ptr + ? aux_input_to_output_row_sums + n_cell + : input_to_output_row_sums + n_cell; + recurrent_to_forget_row_sums = use_cifg + ? recurrent_to_input_row_sums + : recurrent_to_input_row_sums + n_cell; + recurrent_to_cell_row_sums = recurrent_to_forget_row_sums + n_cell; + recurrent_to_output_row_sums = recurrent_to_cell_row_sums + n_cell; + if (projection_weights_ptr != nullptr) { + projection_weights_row_sums = recurrent_to_output_row_sums + n_cell; + } + if (*compute_row_sums) { + ComputeRowSums( + input_to_input_row_sums, input_to_forget_row_sums, + input_to_cell_row_sums, input_to_output_row_sums, + aux_input_to_input_row_sums, aux_input_to_forget_row_sums, + aux_input_to_cell_row_sums, aux_input_to_output_row_sums, + recurrent_to_input_row_sums, recurrent_to_forget_row_sums, + recurrent_to_cell_row_sums, recurrent_to_output_row_sums, + projection_weights_row_sums, row_sums, n_cell, n_input, n_aux_input, + n_output, input_to_input_weights_ptr, input_to_forget_weights_ptr, + input_to_cell_weights_ptr, input_to_output_weights_ptr, + aux_input_to_input_weights_ptr, aux_input_to_forget_weights_ptr, + aux_input_to_cell_weights_ptr, aux_input_to_output_weights_ptr, + recurrent_to_input_weights_ptr, recurrent_to_forget_weights_ptr, + recurrent_to_cell_weights_ptr, recurrent_to_output_weights_ptr, + projection_weights_ptr, use_cifg, aux_input_ptr); + *compute_row_sums = false; + } + } + + // Check if inputs are all zeros so we can skip some computations. + const bool is_input_all_zeros = + micro_tensor_utils::IsZeroVector(input_ptr, n_batch * n_input); + const bool is_aux_input_all_zeros = + (aux_input_ptr == nullptr || + micro_tensor_utils::IsZeroVector(aux_input_ptr, n_batch * n_aux_input)); + const bool is_output_state_all_zeros = + micro_tensor_utils::IsZeroVector(output_state_ptr, n_batch * n_output); + // Quantize inputs. + if (!is_input_all_zeros) { + micro_tensor_utils::BatchQuantizeFloats( + input_ptr, n_batch, n_input, quantized_input_ptr, input_sf, input_zp, + asymmetric_quantize_inputs); + } + if (!is_aux_input_all_zeros) { + micro_tensor_utils::BatchQuantizeFloats( + aux_input_ptr, n_batch, n_aux_input, quantized_aux_input_ptr, + aux_input_sf, aux_input_zp, asymmetric_quantize_inputs); + } + if (!is_output_state_all_zeros) { + micro_tensor_utils::BatchQuantizeFloats( + output_state_ptr, n_batch, n_output, quantized_output_state_ptr, + output_state_sf, output_state_zp, asymmetric_quantize_inputs); + } + if (!use_cifg) { + // Calculate the input gate. (If not CIFG.) + CalculateLstmGateHybrid( + quantized_input_ptr, input_sf, input_zp, input_to_input_weights_ptr, + input_to_input_weights_ledger_ptr, input_to_input_weights_scale, + input_to_input_row_sums, quantized_aux_input_ptr, aux_input_sf, + aux_input_zp, aux_input_to_input_weights_ptr, + aux_input_to_input_weights_scale, aux_input_to_input_row_sums, + quantized_output_state_ptr, output_state_sf, output_state_zp, + recurrent_to_input_weights_ptr, recurrent_to_input_weights_ledger_ptr, + recurrent_to_input_weights_scale, recurrent_to_input_row_sums, + cell_state_ptr, cell_to_input_weights_ptr, cell_to_input_weights_scale, + input_layer_norm_coefficients_ptr, input_gate_bias_ptr, n_batch, + n_input, n_aux_input, n_output, n_cell, kTfLiteActSigmoid, + input_gate_scratch, is_input_all_zeros, is_aux_input_all_zeros, + is_output_state_all_zeros, compute_row_sums, scaling_factors_scratch, + recovered_cell_weights, scales, accum_scratch_ptr); + } + // Calculate the forget gate. + CalculateLstmGateHybrid( + quantized_input_ptr, input_sf, input_zp, input_to_forget_weights_ptr, + input_to_forget_weights_ledger_ptr, input_to_forget_weights_scale, + input_to_forget_row_sums, quantized_aux_input_ptr, aux_input_sf, + aux_input_zp, aux_input_to_forget_weights_ptr, + aux_input_to_forget_weights_scale, aux_input_to_forget_row_sums, + quantized_output_state_ptr, output_state_sf, output_state_zp, + recurrent_to_forget_weights_ptr, recurrent_to_forget_weights_ledger_ptr, + recurrent_to_forget_weights_scale, recurrent_to_forget_row_sums, + cell_state_ptr, cell_to_forget_weights_ptr, cell_to_forget_weights_scale, + forget_layer_norm_coefficients_ptr, forget_gate_bias_ptr, n_batch, + n_input, n_aux_input, n_output, n_cell, kTfLiteActSigmoid, + forget_gate_scratch, is_input_all_zeros, is_aux_input_all_zeros, + is_output_state_all_zeros, compute_row_sums, scaling_factors_scratch, + recovered_cell_weights, scales, accum_scratch_ptr); + // Calculate the cell update gate. + CalculateLstmGateHybrid( + quantized_input_ptr, input_sf, input_zp, input_to_cell_weights_ptr, + input_to_cell_weights_ledger_ptr, input_to_cell_weights_scale, + input_to_cell_row_sums, quantized_aux_input_ptr, aux_input_sf, + aux_input_zp, aux_input_to_cell_weights_ptr, + aux_input_to_cell_weights_scale, aux_input_to_cell_row_sums, + quantized_output_state_ptr, output_state_sf, output_state_zp, + recurrent_to_cell_weights_ptr, recurrent_to_cell_weights_ledger_ptr, + recurrent_to_cell_weights_scale, recurrent_to_cell_row_sums, + /*cell_state=*/nullptr, /*cell_to_gate_weights=*/nullptr, + /*cell_to_gate_weights_scale=*/0.0f, cell_layer_norm_coefficients_ptr, + cell_gate_bias_ptr, n_batch, n_input, n_aux_input, n_output, n_cell, + params->activation, cell_gate_scratch, is_input_all_zeros, + is_aux_input_all_zeros, is_output_state_all_zeros, compute_row_sums, + scaling_factors_scratch, recovered_cell_weights, scales, + accum_scratch_ptr); + // Update the cell state. + UpdateLstmCellFloat(n_batch, n_cell, cell_state_ptr, input_gate_scratch, + forget_gate_scratch, cell_gate_scratch, use_cifg, + params->cell_clip); + // Calculate the output gate. + CalculateLstmGateHybrid( + quantized_input_ptr, input_sf, input_zp, input_to_output_weights_ptr, + input_to_output_weights_ledger_ptr, input_to_output_weights_scale, + input_to_output_row_sums, quantized_aux_input_ptr, aux_input_sf, + aux_input_zp, aux_input_to_output_weights_ptr, + aux_input_to_output_weights_scale, aux_input_to_output_row_sums, + quantized_output_state_ptr, output_state_sf, output_state_zp, + recurrent_to_output_weights_ptr, recurrent_to_output_weights_ledger_ptr, + recurrent_to_output_weights_scale, recurrent_to_output_row_sums, + cell_state_ptr, cell_to_output_weights_ptr, cell_to_output_weights_scale, + output_layer_norm_coefficients_ptr, output_gate_bias_ptr, n_batch, + n_input, n_aux_input, n_output, n_cell, kTfLiteActSigmoid, + output_gate_scratch, is_input_all_zeros, is_aux_input_all_zeros, + is_output_state_all_zeros, compute_row_sums, scaling_factors_scratch, + recovered_cell_weights, scales, accum_scratch_ptr); + // Update the output state. + CalculateLstmOutputHybrid( + n_batch, n_cell, n_output, cell_state_ptr, output_gate_scratch, + params->activation, projection_weights_ptr, projection_weights_ledger_ptr, + projection_weights_scale, projection_bias_ptr, params->proj_clip, + output_state_ptr, asymmetric_quantize_inputs, projection_weights_row_sums, + compute_row_sums, scratch2, quantized_output_scratch, input_sf, input_zp, + accum_scratch_ptr, scales); + // Copy output state to the output. Note that the output's rows may not be + // contiguous (output_batch_leading_dim != n_output). + for (int b = 0; b < n_batch; b++) { + std::memcpy(output_ptr + b * output_batch_leading_dim, + output_state_ptr + b * n_output, n_output * sizeof(float)); + } +} + +// Fully quantized lstm kernel for 16 bit gate matmul output. +// +// Input tensor of size n_batch * n_input: +// input_ptr +// +// LSTM weights: +// Quantized input weights of size 'n_cell * n_input': +// input_to_input_weight_ptr - optional +// input_to_forget_weight_ptr - optional +// input_to_cell_weight_ptr - optional +// input_to_output_weight_ptr - optional +// +// Quantized recurrent weights of size 'n_cell * n_output': +// recurrent_to_input_weight_ptr - optional +// recurrent_to_forget_weights_ptr +// recurrent_to_cell_weights_ptr +// recurrent_to_input_weights_ptr +// +// Quantized peephole weights of size 'n_cell', representing diagonal matrices. +// cell_to_input_weights - optional +// cell_to_cell_weights - optional +// cell_to_output_weights - optional +// +// Quantized projection weights of size 'n_output * n_cell' +// projection_weight_ptr - optional +// +// Weight scales (scalars) for each of the weights above. +// effective_input_to_input_scale_a - optional +// effective_input_to_input_scale_b - optional +// effective_input_to_forget_scale_a +// effective_input_to_forget_scale_b +// effective_input_to_cell_scale_a +// effective_input_to_cell_scale_b +// effective_input_to_output_scale_a +// effective_input_to_output_scale_b +// effective_recurrent_to_input_scale_a - optional +// effective_recurrent_to_input_scale_b - optional +// effective_recurrent_to_forget_scale_a +// effective_recurrent_to_forget_scale_b +// effective_recurrent_to_cell_scale_a +// effective_recurrent_to_cell_scale_b +// effective_recurrent_to_output_scale_a +// effective_recurrent_to_output_scale_b +// effective_proj_scale_a - optional +// effective_proj_scale_b - optional +// +// Gate biases of size 'n_cell': +// input_gate_bias_ptr - optional +// forget_gate_bias_ptr +// cell_gate_bias_ptr +// output_gate_bias_ptr +// +// Layer norm coefficients of size 'n_cell', representing diagonal matrices. +// layer_norm_input_weight_ptr - optional +// layer_norm_forget_weight_ptr - optional +// layer_norm_cell_weight_ptr - optional +// layer_norm_output_weight_ptr - optional +// +// Layer norm scales of size 'n_cell'. +// layer_norm_input_scale_a - optional +// layer_norm_input_scale_b - optional +// layer_norm_forget_scale_a - optional +// layer_norm_forget_scale_b - optional +// layer_norm_cell_scale_a - optional +// layer_norm_cell_scale_b - optional +// layer_norm_output_scale_a - optional +// layer_norm_output_scale_b - optional +// +// Scalar values: +// quantized_cell_clip: quantized clip value for cell. +// quantized_proj_clip: quantized clip value for projection. +// cell_state_scale: the power of two scale for cell state. +// +// Zero points: +// output_state_zp: zero point of output state +// hidden_zp: zero point for hidden state. +// +// Temporary pre-allocated storage for the calculation. Each is of size n_cell * +// n_batch. +// scratch0 +// scratch1 +// scratch2 +// scratch3 +// scratch4 +// scratch5: this scratch buffer is created purely for optimizing the +// MatrixBatchVectorMultiplyAccumulate. +// +// Outputs: +// output_state_ptr - size 'n_batch * n_output' +// cell_state_ptr - size 'n_batch * n_cell' +// output_ptr - size 'n_batch * n_output' +// TODO(b/159947023): scratch0 is not used if (!cifg). Don't allocate then. +inline void LstmStepInteger8x8_16( + const int8_t* input_ptr, const int8_t* input_to_input_weight_ptr, + int32_t effective_input_to_input_scale_a, + int32_t effective_input_to_input_scale_b, + const int8_t* input_to_forget_weight_ptr, + int32_t effective_input_to_forget_scale_a, + int32_t effective_input_to_forget_scale_b, + const int8_t* input_to_cell_weight_ptr, + int32_t effective_input_to_cell_scale_a, + int32_t effective_input_to_cell_scale_b, + const int8_t* input_to_output_weight_ptr, + int32_t effective_input_to_output_scale_a, + int32_t effective_input_to_output_scale_b, + const int8_t* recurrent_to_input_weight_ptr, + int32_t effective_recurrent_to_input_scale_a, + int32_t effective_recurrent_to_input_scale_b, + const int8_t* recurrent_to_forget_weight_ptr, + int32_t effective_recurrent_to_forget_scale_a, + int32_t effective_recurrent_to_forget_scale_b, + const int8_t* recurrent_to_cell_weight_ptr, + int32_t effective_recurrent_to_cell_scale_a, + int32_t effective_recurrent_to_cell_scale_b, + const int8_t* recurrent_to_output_weight_ptr, + int32_t effective_recurrent_to_output_scale_a, + int32_t effective_recurrent_to_output_scale_b, + const int16_t* cell_to_input_weight_ptr, + int32_t effective_cell_to_input_scale_a, + int32_t effective_cell_to_input_scale_b, + const int16_t* cell_to_forget_weight_ptr, + int32_t effective_cell_to_forget_scale_a, + int32_t effective_cell_to_forget_scale_b, + const int16_t* cell_to_output_weight_ptr, + int32_t effective_cell_to_output_scale_a, + int32_t effective_cell_to_output_scale_b, + const int8_t* projection_weight_ptr, int32_t effective_proj_scale_a, + int32_t effective_proj_scale_b, int32_t hidden_zp, + int32_t effective_hidden_scale_a, int32_t effective_hidden_scale_b, + const int16_t* layer_norm_input_weight_ptr, + int32_t layer_norm_input_scale_a, int32_t layer_norm_input_scale_b, + const int16_t* layer_norm_forget_weight_ptr, + int32_t layer_norm_forget_scale_a, int32_t layer_norm_forget_scale_b, + const int16_t* layer_norm_cell_weight_ptr, int32_t layer_norm_cell_scale_a, + int32_t layer_norm_cell_scale_b, + const int16_t* layer_norm_output_weight_ptr, + int32_t layer_norm_output_scale_a, int32_t layer_norm_output_scale_b, + const int32_t* input_gate_bias_ptr, const int32_t* forget_gate_bias_ptr, + const int32_t* cell_gate_bias_ptr, const int32_t* output_gate_bias_ptr, + int16_t quantized_cell_clip, int8_t quantized_proj_clip, + int32_t cell_state_scale, int32_t input_variance_guard, + int32_t forget_variance_guard, int32_t cell_variance_guard, + int32_t output_variance_guard, + const int32_t* input_to_forget_effective_bias, + const int32_t* recurrent_to_forget_effective_bias, + const int32_t* input_to_cell_effective_bias, + const int32_t* recurrent_to_cell_effective_bias, + const int32_t* input_to_output_effective_bias, + const int32_t* recurrent_to_output_effective_bias, + const int32_t* input_to_input_effective_bias, + const int32_t* recurrent_to_input_effective_bias, + const int32_t* projection_effective_bias, int n_batch, int n_cell, + int n_input, int n_output, int8_t* output_state_ptr, + int32_t output_state_zp, int16_t* cell_state_ptr, int8_t* output_ptr, + int16_t* scratch0, int16_t* scratch1, int16_t* scratch2, int16_t* scratch3, + int8_t* scratch4, int32_t* scratch5) { + // Make named scratch buffers for the different gates. + int16_t* input_gate_scratch = scratch0; + int16_t* forget_gate_scratch = scratch1; + int16_t* cell_gate_scratch = scratch2; + int16_t* output_gate_scratch = scratch3; + + // Since we have already checked that weights are all there or none, we + // can check the existence of only one to the get the condition. + const bool use_cifg = (input_to_input_weight_ptr == nullptr); + + // Check for nullptrs. + TFLITE_DCHECK(input_to_forget_effective_bias); + TFLITE_DCHECK(recurrent_to_forget_effective_bias); + TFLITE_DCHECK(input_to_cell_effective_bias); + TFLITE_DCHECK(recurrent_to_cell_effective_bias); + TFLITE_DCHECK(input_to_output_effective_bias); + TFLITE_DCHECK(recurrent_to_output_effective_bias); + if (!use_cifg) { + TFLITE_DCHECK(input_to_input_effective_bias); + TFLITE_DCHECK(recurrent_to_input_effective_bias); + } + const bool use_projection = (projection_weight_ptr != nullptr); + if (use_projection) { + TFLITE_DCHECK(projection_effective_bias); + } + if (!use_cifg) { + // Calculate the input gate. (If not CIFG.) + CalculateLstmGateInteger8x8_16( + input_ptr, input_to_input_weight_ptr, input_to_input_effective_bias, + effective_input_to_input_scale_a, effective_input_to_input_scale_b, + output_state_ptr, recurrent_to_input_weight_ptr, + recurrent_to_input_effective_bias, effective_recurrent_to_input_scale_a, + effective_recurrent_to_input_scale_b, cell_state_ptr, + cell_to_input_weight_ptr, effective_cell_to_input_scale_a, + effective_cell_to_input_scale_b, layer_norm_input_weight_ptr, + input_gate_bias_ptr, layer_norm_input_scale_a, layer_norm_input_scale_b, + input_variance_guard, n_batch, n_input, n_output, n_cell, + kTfLiteActSigmoid, input_gate_scratch, scratch5); + } + // Calculate the forget gate. + CalculateLstmGateInteger8x8_16( + input_ptr, input_to_forget_weight_ptr, input_to_forget_effective_bias, + effective_input_to_forget_scale_a, effective_input_to_forget_scale_b, + output_state_ptr, recurrent_to_forget_weight_ptr, + recurrent_to_forget_effective_bias, effective_recurrent_to_forget_scale_a, + effective_recurrent_to_forget_scale_b, cell_state_ptr, + cell_to_forget_weight_ptr, effective_cell_to_forget_scale_a, + effective_cell_to_forget_scale_b, layer_norm_forget_weight_ptr, + forget_gate_bias_ptr, layer_norm_forget_scale_a, + layer_norm_forget_scale_b, forget_variance_guard, n_batch, n_input, + n_output, n_cell, kTfLiteActSigmoid, forget_gate_scratch, scratch5); + // Calculate the cell update gate. + CalculateLstmGateInteger8x8_16( + input_ptr, input_to_cell_weight_ptr, input_to_cell_effective_bias, + effective_input_to_cell_scale_a, effective_input_to_cell_scale_b, + output_state_ptr, recurrent_to_cell_weight_ptr, + recurrent_to_cell_effective_bias, effective_recurrent_to_cell_scale_a, + effective_recurrent_to_cell_scale_b, cell_state_ptr, + /*cell_to_gate_weights=*/nullptr, /*cell_to_gate_scale_a=*/0, + /*cell_to_gate_scale_b=*/0, layer_norm_cell_weight_ptr, + cell_gate_bias_ptr, layer_norm_cell_scale_a, layer_norm_cell_scale_b, + cell_variance_guard, n_batch, n_input, n_output, n_cell, kTfLiteActTanh, + cell_gate_scratch, scratch5); + // Update the cell state. + UpdateLstmCellInteger(n_batch, n_cell, cell_state_ptr, cell_state_scale, + input_gate_scratch, forget_gate_scratch, + cell_gate_scratch, use_cifg, quantized_cell_clip); + // Calculate the output gate. + CalculateLstmGateInteger8x8_16( + input_ptr, input_to_output_weight_ptr, input_to_output_effective_bias, + effective_input_to_output_scale_a, effective_input_to_output_scale_b, + output_state_ptr, recurrent_to_output_weight_ptr, + recurrent_to_output_effective_bias, effective_recurrent_to_output_scale_a, + effective_recurrent_to_output_scale_b, cell_state_ptr, + cell_to_output_weight_ptr, effective_cell_to_output_scale_a, + effective_cell_to_output_scale_b, layer_norm_output_weight_ptr, + output_gate_bias_ptr, layer_norm_output_scale_a, + layer_norm_output_scale_b, output_variance_guard, n_batch, n_input, + n_output, n_cell, kTfLiteActSigmoid, output_gate_scratch, scratch5); + // Update the output state. + CalculateLstmOutputInteger8x8_16( + n_batch, n_cell, n_output, cell_state_ptr, cell_state_scale, + output_gate_scratch, effective_hidden_scale_a, effective_hidden_scale_b, + hidden_zp, projection_weight_ptr, effective_proj_scale_a, + effective_proj_scale_b, projection_effective_bias, output_state_zp, + quantized_proj_clip, output_state_ptr, scratch0, scratch4, scratch5); + // Copy output state to the output. Note that unlike float or hybrid, output + // is always contiguous. + std::memcpy(output_ptr, output_state_ptr, + n_batch * n_output * sizeof(int8_t)); +} + +// Fully quantized lstm kernel for 8 bit gate matmul output. +// +// Input tensor of size n_batch * n_input: +// input_ptr +// +// LSTM weights: +// Quantized input weights of size 'n_cell * n_input': +// input_to_input_weight_ptr - optional +// input_to_forget_weight_ptr - optional +// input_to_cell_weight_ptr - optional +// input_to_output_weight_ptr - optional +// +// Quantized recurrent weights of size 'n_cell * n_output': +// recurrent_to_input_weight_ptr - optional +// recurrent_to_forget_weights_ptr +// recurrent_to_cell_weights_ptr +// recurrent_to_input_weights_ptr +// +// Quantized peephole weights of size 'n_cell', representing diagonal matrices. +// cell_to_input_weights - optional +// cell_to_cell_weights - optional +// cell_to_output_weights - optional +// +// Quantized projection weights of size 'n_output * n_cell' +// projection_weight_ptr - optional +// +// Weight scales (scalars) for each of the weights above. +// effective_input_to_input_scale_a - optional +// effective_input_to_input_scale_b - optional +// effective_input_to_forget_scale_a +// effective_input_to_forget_scale_b +// effective_input_to_cell_scale_a +// effective_input_to_cell_scale_b +// effective_input_to_output_scale_a +// effective_input_to_output_scale_b +// effective_recurrent_to_input_scale_a - optional +// effective_recurrent_to_input_scale_b - optional +// effective_recurrent_to_forget_scale_a +// effective_recurrent_to_forget_scale_b +// effective_recurrent_to_cell_scale_a +// effective_recurrent_to_cell_scale_b +// effective_recurrent_to_output_scale_a +// effective_recurrent_to_output_scale_b +// effective_proj_scale_a - optional +// effective_proj_scale_b - optional +// +// Gate biases of size 'n_cell': +// input_gate_bias_ptr - optional +// forget_gate_bias_ptr +// cell_gate_bias_ptr +// output_gate_bias_ptr +// +// Layer norm coefficients of size 'n_cell', representing diagonal matrices. +// layer_norm_input_weight_ptr - optional +// layer_norm_forget_weight_ptr - optional +// layer_norm_cell_weight_ptr - optional +// layer_norm_output_weight_ptr - optional +// +// Layer norm scales of size 'n_cell'. +// layer_norm_input_scale_a - optional +// layer_norm_input_scale_b - optional +// layer_norm_forget_scale_a - optional +// layer_norm_forget_scale_b - optional +// layer_norm_cell_scale_a - optional +// layer_norm_cell_scale_b - optional +// layer_norm_output_scale_a - optional +// layer_norm_output_scale_b - optional +// +// Scalar values: +// quantized_cell_clip: quantized clip value for cell. +// quantized_proj_clip: quantized clip value for projection. +// cell_state_scale: the power of two scale for cell state. +// +// Zero points: +// input_zp: zero point for input tensor. +// output_state_zp: zero point of output state. +// hidden_zp: zero point for hidden state. +// +// Temporary pre-allocated storage for the calculation. Each is of size n_cell * +// n_batch. +// scratch0 +// scratch1 +// scratch2 +// scratch3 +// scratch4 +// scratch5 +// scratch6 +// scratch7 +// +// Outputs: +// output_state_ptr - size 'n_batch * n_output' +// cell_state_ptr - size 'n_batch * n_cell' +// output_ptr - size 'n_batch * n_output' +// +// Can move zero point calculation into Prepare() for better perfomance. +// TODO(b/159947023): scratch5 is unused, remove. +inline void LstmStepInteger8x8_8( + const int8_t* input_ptr, int32_t input_zp, + const int8_t* input_to_input_weight_ptr, + int32_t effective_input_to_input_scale_a, + int32_t effective_input_to_input_scale_b, + const int8_t* input_to_forget_weight_ptr, + int32_t effective_input_to_forget_scale_a, + int32_t effective_input_to_forget_scale_b, + const int8_t* input_to_cell_weight_ptr, + int32_t effective_input_to_cell_scale_a, + int32_t effective_input_to_cell_scale_b, + const int8_t* input_to_output_weight_ptr, + int32_t effective_input_to_output_scale_a, + int32_t effective_input_to_output_scale_b, + const int8_t* recurrent_to_input_weight_ptr, + int32_t effective_recurrent_to_input_scale_a, + int32_t effective_recurrent_to_input_scale_b, + const int8_t* recurrent_to_forget_weight_ptr, + int32_t effective_recurrent_to_forget_scale_a, + int32_t effective_recurrent_to_forget_scale_b, + const int8_t* recurrent_to_cell_weight_ptr, + int32_t effective_recurrent_to_cell_scale_a, + int32_t effective_recurrent_to_cell_scale_b, + const int8_t* recurrent_to_output_weight_ptr, + int32_t effective_recurrent_to_output_scale_a, + int32_t effective_recurrent_to_output_scale_b, + const int8_t* cell_to_input_weight_ptr, + int32_t effective_cell_to_input_scale_a, + int32_t effective_cell_to_input_scale_b, + const int8_t* cell_to_forget_weight_ptr, + int32_t effective_cell_to_forget_scale_a, + int32_t effective_cell_to_forget_scale_b, + const int8_t* cell_to_output_weight_ptr, + int32_t effective_cell_to_output_scale_a, + int32_t effective_cell_to_output_scale_b, + const int8_t* projection_weight_ptr, int32_t effective_proj_scale_a, + int32_t effective_proj_scale_b, const int16_t* layer_norm_input_weight_ptr, + int32_t layer_norm_input_scale_a, int32_t layer_norm_input_scale_b, + const int16_t* layer_norm_forget_weight_ptr, + int32_t layer_norm_forget_scale_a, int32_t layer_norm_forget_scale_b, + const int16_t* layer_norm_cell_weight_ptr, int32_t layer_norm_cell_scale_a, + int32_t layer_norm_cell_scale_b, + const int16_t* layer_norm_output_weight_ptr, + int32_t layer_norm_output_scale_a, int32_t layer_norm_output_scale_b, + const int32_t* input_gate_bias_ptr, const int32_t* forget_gate_bias_ptr, + const int32_t* cell_gate_bias_ptr, const int32_t* output_gate_bias_ptr, + const int32_t* projection_bias_ptr, const TfLiteLSTMParams* params, + const int32_t* intermediate_scale_a, const int32_t* intermediate_scale_b, + const int32_t* intermediate_zp, int16_t quantized_cell_clip, + int8_t quantized_proj_clip, int n_batch, int n_cell, int n_input, + int n_output, int output_batch_leading_dim, int8_t* output_state_ptr, + int32_t output_state_zp, int16_t* cell_state_ptr, int8_t* output_ptr, + int8_t* scratch0, int8_t* scratch1, int16_t* scratch2, int16_t* scratch3, + int16_t* scratch4, int16_t* scratch5, int16_t* scratch6, + int16_t* scratch7) { + // TODO(b/159066113): scratch5 is unused, remove. + + // Make named scratch buffers for the different gates. + int16_t* forget_gate_scratch = scratch2; + int16_t* cell_gate_scratch = scratch3; + int16_t* output_gate_scratch = scratch4; + // no-CIFG is not supported here + + // Calculate the forget gate. + CalculateLstmGateInteger8x8_8( + input_ptr, input_zp, input_to_forget_weight_ptr, + effective_input_to_forget_scale_a, effective_input_to_forget_scale_b, + intermediate_scale_a[2], intermediate_scale_b[2], intermediate_zp[4], + output_state_ptr, output_state_zp, recurrent_to_forget_weight_ptr, + effective_recurrent_to_forget_scale_a, + effective_recurrent_to_forget_scale_b, intermediate_scale_a[3], + intermediate_scale_b[3], intermediate_zp[5], layer_norm_forget_weight_ptr, + layer_norm_forget_scale_a, layer_norm_forget_scale_b, + forget_gate_bias_ptr, n_batch, n_input, n_output, n_cell, + kTfLiteActSigmoid, forget_gate_scratch, scratch0, scratch1); + // Calculate the cell update gate. + CalculateLstmGateInteger8x8_8( + input_ptr, input_zp, input_to_cell_weight_ptr, + effective_input_to_cell_scale_a, effective_input_to_cell_scale_b, + intermediate_scale_a[4], intermediate_scale_b[4], intermediate_zp[7], + output_state_ptr, output_state_zp, recurrent_to_cell_weight_ptr, + effective_recurrent_to_cell_scale_a, effective_recurrent_to_cell_scale_b, + intermediate_scale_a[5], intermediate_scale_b[5], intermediate_zp[8], + layer_norm_cell_weight_ptr, layer_norm_cell_scale_a, + layer_norm_cell_scale_b, cell_gate_bias_ptr, n_batch, n_input, n_output, + n_cell, kTfLiteActTanh, cell_gate_scratch, scratch0, scratch1); + // Update the cell state. + UpdateLstmCellInteger(n_batch, n_cell, cell_state_ptr, + /*cell_state_scale=*/-15, /*input_gate=*/nullptr, + forget_gate_scratch, cell_gate_scratch, + /*use_cifg=*/true, quantized_cell_clip); + // Calculate the output gate. + CalculateLstmGateInteger8x8_8( + input_ptr, input_zp, input_to_output_weight_ptr, + effective_input_to_output_scale_a, effective_input_to_output_scale_b, + intermediate_scale_a[6], intermediate_scale_b[6], intermediate_zp[10], + output_state_ptr, output_state_zp, recurrent_to_output_weight_ptr, + effective_recurrent_to_output_scale_a, + effective_recurrent_to_output_scale_b, intermediate_scale_a[11], + intermediate_scale_b[7], intermediate_zp[7], layer_norm_output_weight_ptr, + layer_norm_output_scale_a, layer_norm_output_scale_b, + output_gate_bias_ptr, n_batch, n_input, n_output, n_cell, + kTfLiteActSigmoid, output_gate_scratch, scratch0, scratch1); + // Update the output state. + CalculateLstmOutputInteger8x8_8( + n_batch, n_cell, n_output, cell_state_ptr, output_gate_scratch, + projection_weight_ptr, effective_proj_scale_a, effective_proj_scale_b, + projection_bias_ptr, output_state_zp, quantized_proj_clip, + output_state_ptr, scratch2); + // Copy output state to the output. Note that unlike float or hybrid, output + // is always contigous. + std::memcpy(output_ptr, output_state_ptr, + n_batch * n_output * sizeof(int8_t)); +} + +} // namespace + +TfLiteStatus EvalFloatLstm( + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* input_to_input_weights, + const TfLiteEvalTensor* input_to_forget_weights, + const TfLiteEvalTensor* input_to_cell_weights, + const TfLiteEvalTensor* input_to_output_weights, + const TfLiteEvalTensor* recurrent_to_input_weights, + const TfLiteEvalTensor* recurrent_to_forget_weights, + const TfLiteEvalTensor* recurrent_to_cell_weights, + const TfLiteEvalTensor* recurrent_to_output_weights, + const TfLiteEvalTensor* cell_to_input_weights, + const TfLiteEvalTensor* cell_to_forget_weights, + const TfLiteEvalTensor* cell_to_output_weights, + const TfLiteEvalTensor* input_layer_norm_coefficients, + const TfLiteEvalTensor* forget_layer_norm_coefficients, + const TfLiteEvalTensor* cell_layer_norm_coefficients, + const TfLiteEvalTensor* output_layer_norm_coefficients, + const TfLiteEvalTensor* aux_input, + const TfLiteEvalTensor* aux_input_to_input_weights, + const TfLiteEvalTensor* aux_input_to_forget_weights, + const TfLiteEvalTensor* aux_input_to_cell_weights, + const TfLiteEvalTensor* aux_input_to_output_weights, + const TfLiteEvalTensor* input_gate_bias, + const TfLiteEvalTensor* forget_gate_bias, + const TfLiteEvalTensor* cell_gate_bias, + const TfLiteEvalTensor* output_gate_bias, + const TfLiteEvalTensor* projection_weights, + const TfLiteEvalTensor* projection_bias, const TfLiteLSTMParams* params, + bool forward_sequence, bool time_major, int output_offset, + float* scratch_buffer, TfLiteEvalTensor* output_state, + TfLiteEvalTensor* cell_state, TfLiteEvalTensor* output) { + TFLITE_DCHECK(input->dims->size >= 2 && input->dims->size <= 3); + int max_time, n_batch; + if (input->dims->size == 3) { + max_time = (time_major) ? input->dims->data[0] : input->dims->data[1]; + n_batch = (time_major) ? input->dims->data[1] : input->dims->data[0]; + } else { + max_time = 1; + n_batch = input->dims->data[0]; + } + const int n_input = input->dims->data[input->dims->size - 1]; + const int aux_input_size = + (aux_input) ? aux_input->dims->data[aux_input->dims->size - 1] : 0; + + // n_cell and n_output will be the same size when there is no projection. + const int n_cell = input_to_output_weights->dims->data[0]; + const int n_output = recurrent_to_output_weights->dims->data[1]; + + // Since we have already checked that weights are all there or none, we can + // check the existence of only one to the get the condition. + const bool use_cifg = (input_to_input_weights == nullptr); + + // Index the scratch buffers pointers to the global scratch buffer. + float* input_gate_scratch = nullptr; + float* cell_gate_scratch = nullptr; + float* forget_gate_scratch = nullptr; + float* output_gate_scratch = nullptr; + if (use_cifg) { + cell_gate_scratch = scratch_buffer; + forget_gate_scratch = scratch_buffer + n_cell * n_batch; + output_gate_scratch = scratch_buffer + 2 * n_cell * n_batch; + } else { + input_gate_scratch = scratch_buffer; + cell_gate_scratch = scratch_buffer + n_cell * n_batch; + forget_gate_scratch = scratch_buffer + 2 * n_cell * n_batch; + output_gate_scratch = scratch_buffer + 3 * n_cell * n_batch; + } + + const int output_batch_leading_dim = + output->dims->data[output->dims->size - 1]; + if (time_major) { + // Loop through the sequence. + const int input_step = n_batch * n_input; + const int output_step = n_batch * output_batch_leading_dim; + for (int t = 0; t < max_time; t++) { + // If this is the forward_sequence, step forward, otherwise step + // backwards. + const int t_rel = forward_sequence ? t : max_time - t - 1; + const float* input_ptr = + tflite::micro::GetTensorData(input) + t_rel * input_step; + const float* aux_input_ptr = nullptr; + if (aux_input) { + aux_input_ptr = + tflite::micro::GetTensorData(aux_input) + t_rel * input_step; + } + float* output_ptr = tflite::micro::GetTensorData(output) + + t_rel * output_step + output_offset; + + LstmStepFloat( + input_ptr, + input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_input_weights), + input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_forget_weights), + input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_cell_weights), + input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_output_weights), + aux_input_ptr, + aux_input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(aux_input_to_input_weights), + aux_input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_forget_weights), + aux_input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(aux_input_to_cell_weights), + aux_input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_output_weights), + recurrent_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(recurrent_to_input_weights), + recurrent_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_forget_weights), + recurrent_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(recurrent_to_cell_weights), + recurrent_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_output_weights), + cell_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_input_weights), + cell_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_forget_weights), + cell_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_output_weights), + input_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_layer_norm_coefficients), + forget_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + forget_layer_norm_coefficients), + cell_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + cell_layer_norm_coefficients), + output_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + output_layer_norm_coefficients), + input_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_gate_bias), + forget_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(forget_gate_bias), + cell_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_gate_bias), + output_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(output_gate_bias), + projection_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_weights), + projection_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_bias), + params, n_batch, n_cell, n_input, aux_input_size, n_output, + output_batch_leading_dim, + tflite::micro::GetTensorData(output_state), + tflite::micro::GetTensorData(cell_state), input_gate_scratch, + forget_gate_scratch, cell_gate_scratch, output_gate_scratch, + output_ptr); + } + } else { + for (int b = 0; b < n_batch; b++) { + const int input_step = n_input; + const int output_step = output_batch_leading_dim; + for (int t = 0; t < max_time; t++) { + // If this is the forward_sequence, step forward, otherwise step + // backwards. + const int t_rel = forward_sequence ? t : max_time - t - 1; + const int time_offset = b * max_time + t_rel; + const float* input_ptr = tflite::micro::GetTensorData(input) + + time_offset * input_step; + const float* aux_input_ptr = nullptr; + if (aux_input) { + aux_input_ptr = tflite::micro::GetTensorData(aux_input) + + time_offset * input_step; + } + float* output_ptr = tflite::micro::GetTensorData(output) + + time_offset * output_step + output_offset; + + // Offset the {output,cell}_state pointers to the right batch. + float* output_state_ptr = + tflite::micro::GetTensorData(output_state) + + b * output_batch_leading_dim; + float* cell_state_ptr = + tflite::micro::GetTensorData(cell_state) + b * n_cell; + // Offset the scratch pointers to the right batch. + float* input_gate_scratch_ptr = + input_gate_scratch ? input_gate_scratch + b * n_cell : nullptr; + float* forget_gate_scratch_ptr = forget_gate_scratch + b * n_cell; + float* cell_gate_scratch_ptr = cell_gate_scratch + b * n_cell; + float* output_gate_scratch_ptr = output_gate_scratch + b * n_cell; + + LstmStepFloat( + input_ptr, + input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_input_weights), + input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_forget_weights), + input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_cell_weights), + input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_output_weights), + aux_input_ptr, + aux_input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_input_weights), + aux_input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_forget_weights), + aux_input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_cell_weights), + aux_input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_output_weights), + recurrent_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_input_weights), + recurrent_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_forget_weights), + recurrent_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_cell_weights), + recurrent_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_output_weights), + cell_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_input_weights), + cell_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_forget_weights), + cell_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_output_weights), + input_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_layer_norm_coefficients), + forget_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + forget_layer_norm_coefficients), + cell_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + cell_layer_norm_coefficients), + output_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + output_layer_norm_coefficients), + input_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_gate_bias), + forget_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(forget_gate_bias), + cell_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_gate_bias), + output_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(output_gate_bias), + projection_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_weights), + projection_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_bias), + params, + /*n_batch=*/1, n_cell, n_input, aux_input_size, n_output, + output_batch_leading_dim, output_state_ptr, cell_state_ptr, + input_gate_scratch_ptr, forget_gate_scratch_ptr, + cell_gate_scratch_ptr, output_gate_scratch_ptr, output_ptr); + } + } + } + return kTfLiteOk; +} + +TfLiteStatus EvalHybridLstm( + const HybridLstmScales* hybrid_lstm_scales, const TfLiteEvalTensor* input, + const TfLiteEvalTensor* input_to_input_weights, + const TfLiteEvalTensor* input_to_input_weights_ledger, + const TfLiteEvalTensor* input_to_forget_weights, + const TfLiteEvalTensor* input_to_forget_weights_ledger, + const TfLiteEvalTensor* input_to_cell_weights, + const TfLiteEvalTensor* input_to_cell_weights_ledger, + const TfLiteEvalTensor* input_to_output_weights, + const TfLiteEvalTensor* input_to_output_weights_ledger, + const TfLiteEvalTensor* recurrent_to_input_weights, + const TfLiteEvalTensor* recurrent_to_input_weights_ledger, + const TfLiteEvalTensor* recurrent_to_forget_weights, + const TfLiteEvalTensor* recurrent_to_forget_weights_ledger, + const TfLiteEvalTensor* recurrent_to_cell_weights, + const TfLiteEvalTensor* recurrent_to_cell_weights_ledger, + const TfLiteEvalTensor* recurrent_to_output_weights, + const TfLiteEvalTensor* recurrent_to_output_weights_ledger, + const TfLiteEvalTensor* cell_to_input_weights, + const TfLiteEvalTensor* cell_to_forget_weights, + const TfLiteEvalTensor* cell_to_output_weights, + const TfLiteEvalTensor* input_layer_norm_coefficients, + const TfLiteEvalTensor* forget_layer_norm_coefficients, + const TfLiteEvalTensor* cell_layer_norm_coefficients, + const TfLiteEvalTensor* output_layer_norm_coefficients, + const TfLiteEvalTensor* aux_input, + const TfLiteEvalTensor* aux_input_to_input_weights, + const TfLiteEvalTensor* aux_input_to_forget_weights, + const TfLiteEvalTensor* aux_input_to_cell_weights, + const TfLiteEvalTensor* aux_input_to_output_weights, + const TfLiteEvalTensor* input_gate_bias, + const TfLiteEvalTensor* forget_gate_bias, + const TfLiteEvalTensor* cell_gate_bias, + const TfLiteEvalTensor* output_gate_bias, + const TfLiteEvalTensor* projection_weights, + const TfLiteEvalTensor* projection_weights_ledger, + const TfLiteEvalTensor* projection_bias, const TfLiteLSTMParams* params, + bool forward_sequence, bool time_major, int output_offset, + float* scratch_buffer, float* input_sf, float* aux_input_sf, + float* output_state_sf, float* prod_scaling_factors, + float* recovered_cell_weights, int8_t* input_quantized, + int8_t* aux_input_quantized, int8_t* output_state_quantized, + int8_t* cell_state_quantized, float* scales, TfLiteEvalTensor* output_state, + TfLiteEvalTensor* cell_state, int32_t* output_scratch_buffer, + TfLiteEvalTensor* output, int32_t* input_zp, int32_t* aux_input_zp, + int32_t* output_state_zp, int32_t* row_sums, int row_sums_size, + bool* compute_row_sums) { + TFLITE_DCHECK(input->dims->size >= 2 && input->dims->size <= 3); + const int n_input = input->dims->data[input->dims->size - 1]; + int max_time, n_batch; + if (input->dims->size == 2) { + max_time = 1; + n_batch = input->dims->data[0]; + } else { + max_time = (time_major) ? input->dims->data[0] : input->dims->data[1]; + n_batch = (time_major) ? input->dims->data[1] : input->dims->data[0]; + } + const int aux_input_size = + (aux_input) ? aux_input->dims->data[aux_input->dims->size - 1] : 0; + // n_cell and n_output will be the same size when there is no projection. + const int n_cell = input_to_output_weights->dims->data[0]; + const int n_output = recurrent_to_output_weights->dims->data[1]; + + // Since we have already checked that weights are all there or none, we can + // check the existence of only one to get the condition. + const bool use_cifg = (input_to_input_weights == nullptr); + + float* input_gate_scratch = nullptr; + float* cell_gate_scratch = nullptr; + float* forget_gate_scratch = nullptr; + float* output_gate_scratch = nullptr; + if (use_cifg) { + cell_gate_scratch = scratch_buffer; + forget_gate_scratch = scratch_buffer + n_cell * n_batch; + output_gate_scratch = scratch_buffer + 2 * n_cell * n_batch; + } else { + input_gate_scratch = scratch_buffer; + cell_gate_scratch = scratch_buffer + n_cell * n_batch; + forget_gate_scratch = scratch_buffer + 2 * n_cell * n_batch; + output_gate_scratch = scratch_buffer + 3 * n_cell * n_batch; + } + + const int output_batch_leading_dim = + output->dims->data[output->dims->size - 1]; + + int32_t* input_zp_ptr = nullptr; + int32_t* aux_input_zp_ptr = nullptr; + int32_t* output_state_zp_ptr = nullptr; + int32_t* row_sums_ptr = nullptr; + if (params->asymmetric_quantize_inputs) { + input_zp_ptr = input_zp; + aux_input_zp_ptr = aux_input_zp; + output_state_zp_ptr = output_state_zp; + row_sums_ptr = row_sums; + } + + if (time_major) { + // Feed the sequence into the LSTM step-by-step. + const int input_step = n_batch * n_input; + const int output_step = n_batch * output_batch_leading_dim; + for (int t = 0; t < max_time; t++) { + // If this is the forward_sequence, step forward, otherwise step + // backwards. + const int t_rel = forward_sequence ? t : max_time - t - 1; + const float* input_ptr = + tflite::micro::GetTensorData(input) + t_rel * input_step; + const float* aux_input_ptr = nullptr; + if (aux_input) { + aux_input_ptr = + tflite::micro::GetTensorData(aux_input) + t_rel * input_step; + } + float* output_ptr = tflite::micro::GetTensorData(output) + + t_rel * output_step + output_offset; + LstmStepHybrid( + input_ptr, + input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_input_weights), + input_to_input_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_to_input_weights_ledger), + hybrid_lstm_scales->input_to_input_weights_scale, + input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_forget_weights), + input_to_forget_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_to_forget_weights_ledger), + hybrid_lstm_scales->input_to_forget_weights_scale, + input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_cell_weights), + input_to_cell_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_to_cell_weights_ledger), + hybrid_lstm_scales->input_to_cell_weights_scale, + input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_output_weights), + input_to_output_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_to_output_weights_ledger), + hybrid_lstm_scales->input_to_output_weights_scale, aux_input_ptr, + aux_input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_input_weights), + hybrid_lstm_scales->aux_input_to_input_weights_scale, + aux_input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_forget_weights), + hybrid_lstm_scales->aux_input_to_forget_weights_scale, + aux_input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(aux_input_to_cell_weights), + hybrid_lstm_scales->aux_input_to_cell_weights_scale, + aux_input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_output_weights), + hybrid_lstm_scales->aux_input_to_output_weights_scale, + recurrent_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_input_weights), + recurrent_to_input_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_input_weights_ledger), + hybrid_lstm_scales->recurrent_to_input_weights_scale, + recurrent_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_forget_weights), + recurrent_to_forget_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_forget_weights_ledger), + hybrid_lstm_scales->recurrent_to_forget_weights_scale, + recurrent_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(recurrent_to_cell_weights), + recurrent_to_cell_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_cell_weights_ledger), + hybrid_lstm_scales->recurrent_to_cell_weights_scale, + recurrent_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_output_weights), + recurrent_to_output_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_output_weights_ledger), + hybrid_lstm_scales->recurrent_to_output_weights_scale, + cell_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_input_weights), + hybrid_lstm_scales->cell_to_input_weights_scale, + cell_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_forget_weights), + hybrid_lstm_scales->cell_to_forget_weights_scale, + cell_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_output_weights), + hybrid_lstm_scales->cell_to_output_weights_scale, + input_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_layer_norm_coefficients), + forget_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + forget_layer_norm_coefficients), + cell_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + cell_layer_norm_coefficients), + output_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + output_layer_norm_coefficients), + input_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_gate_bias), + forget_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(forget_gate_bias), + cell_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_gate_bias), + output_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(output_gate_bias), + projection_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_weights), + projection_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + projection_weights_ledger), + hybrid_lstm_scales->projection_weights_scale, + projection_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_bias), + params, n_batch, n_cell, n_input, aux_input_size, n_output, + output_batch_leading_dim, input_gate_scratch, forget_gate_scratch, + cell_gate_scratch, output_gate_scratch, scales, input_sf, + aux_input_sf, output_state_sf, prod_scaling_factors, + recovered_cell_weights, input_quantized, aux_input_quantized, + output_state_quantized, cell_state_quantized, + tflite::micro::GetTensorData(output_state), + tflite::micro::GetTensorData(cell_state), + output_scratch_buffer, output_ptr, input_zp_ptr, aux_input_zp_ptr, + output_state_zp_ptr, row_sums_ptr, row_sums_size, compute_row_sums, + params->asymmetric_quantize_inputs); + } + } else { + for (int b = 0; b < n_batch; b++) { + const int input_step = n_input; + const int output_step = output_batch_leading_dim; + for (int t = 0; t < max_time; t++) { + // If this is the forward_sequence, step forward, otherwise step + // backwards. + const int t_rel = forward_sequence ? t : max_time - t - 1; + const int time_offset = b * max_time + t_rel; + const float* input_ptr = tflite::micro::GetTensorData(input) + + time_offset * input_step; + const float* aux_input_ptr = nullptr; + if (aux_input) { + aux_input_ptr = tflite::micro::GetTensorData(aux_input) + + time_offset * input_step; + } + float* output_ptr = tflite::micro::GetTensorData(output) + + time_offset * output_step + output_offset; + + // Offset the {output,cell}_state pointers to the right batch. + float* output_state_ptr = + tflite::micro::GetTensorData(output_state) + + b * output_batch_leading_dim; + float* cell_state_ptr = + tflite::micro::GetTensorData(cell_state) + b * n_cell; + // Offset the scratch pointers to the right batch. + float* input_gate_scratch_ptr = + input_gate_scratch ? input_gate_scratch + b * n_cell : nullptr; + float* forget_gate_scratch_ptr = forget_gate_scratch + b * n_cell; + float* cell_gate_scratch_ptr = cell_gate_scratch + b * n_cell; + float* output_gate_scratch_ptr = output_gate_scratch + b * n_cell; + + LstmStepHybrid( + input_ptr, + input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_input_weights), + input_to_input_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_to_input_weights_ledger), + hybrid_lstm_scales->input_to_input_weights_scale, + input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_forget_weights), + input_to_forget_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_to_forget_weights_ledger), + hybrid_lstm_scales->input_to_forget_weights_scale, + input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_cell_weights), + input_to_cell_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_to_cell_weights_ledger), + hybrid_lstm_scales->input_to_cell_weights_scale, + input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_output_weights), + input_to_output_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_to_output_weights_ledger), + hybrid_lstm_scales->input_to_output_weights_scale, aux_input_ptr, + aux_input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_input_weights), + hybrid_lstm_scales->aux_input_to_input_weights_scale, + aux_input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_forget_weights), + hybrid_lstm_scales->aux_input_to_forget_weights_scale, + aux_input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_cell_weights), + hybrid_lstm_scales->aux_input_to_cell_weights_scale, + aux_input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + aux_input_to_output_weights), + hybrid_lstm_scales->aux_input_to_output_weights_scale, + recurrent_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_input_weights), + recurrent_to_input_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_input_weights_ledger), + hybrid_lstm_scales->recurrent_to_input_weights_scale, + recurrent_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_forget_weights), + recurrent_to_forget_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_forget_weights_ledger), + hybrid_lstm_scales->recurrent_to_forget_weights_scale, + recurrent_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_cell_weights), + recurrent_to_cell_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_cell_weights_ledger), + hybrid_lstm_scales->recurrent_to_cell_weights_scale, + recurrent_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_output_weights), + recurrent_to_output_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_output_weights_ledger), + hybrid_lstm_scales->recurrent_to_output_weights_scale, + cell_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_input_weights), + hybrid_lstm_scales->cell_to_input_weights_scale, + cell_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_forget_weights), + hybrid_lstm_scales->cell_to_forget_weights_scale, + cell_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_output_weights), + hybrid_lstm_scales->cell_to_output_weights_scale, + input_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_layer_norm_coefficients), + forget_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + forget_layer_norm_coefficients), + cell_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + cell_layer_norm_coefficients), + output_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + output_layer_norm_coefficients), + input_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_gate_bias), + forget_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(forget_gate_bias), + cell_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_gate_bias), + output_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(output_gate_bias), + projection_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_weights), + projection_weights_ledger == nullptr + ? nullptr + : tflite::micro::GetTensorData( + projection_weights_ledger), + hybrid_lstm_scales->projection_weights_scale, + projection_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_bias), + params, + /*n_batch=*/1, n_cell, n_input, aux_input_size, n_output, + output_batch_leading_dim, input_gate_scratch_ptr, + forget_gate_scratch_ptr, cell_gate_scratch_ptr, + output_gate_scratch_ptr, scales, input_sf, aux_input_sf, + output_state_sf, prod_scaling_factors, recovered_cell_weights, + input_quantized, aux_input_quantized, output_state_quantized, + cell_state_quantized, output_state_ptr, cell_state_ptr, + output_scratch_buffer, output_ptr, input_zp_ptr, aux_input_zp_ptr, + output_state_zp_ptr, row_sums_ptr, row_sums_size, compute_row_sums, + params->asymmetric_quantize_inputs); + } + } + } + + return kTfLiteOk; +} + +TfLiteStatus EvalInteger8x8_16Lstm( + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* input_to_input_weights, + const TfLiteEvalTensor* input_to_forget_weights, + const TfLiteEvalTensor* input_to_cell_weights, + const TfLiteEvalTensor* input_to_output_weights, + const TfLiteEvalTensor* recurrent_to_input_weights, + const TfLiteEvalTensor* recurrent_to_forget_weights, + const TfLiteEvalTensor* recurrent_to_cell_weights, + const TfLiteEvalTensor* recurrent_to_output_weights, + const TfLiteEvalTensor* cell_to_input_weights, + const TfLiteEvalTensor* cell_to_forget_weights, + const TfLiteEvalTensor* cell_to_output_weights, + const TfLiteEvalTensor* input_layer_norm_coefficients, + const TfLiteEvalTensor* forget_layer_norm_coefficients, + const TfLiteEvalTensor* cell_layer_norm_coefficients, + const TfLiteEvalTensor* output_layer_norm_coefficients, + const TfLiteEvalTensor* input_gate_bias, + const TfLiteEvalTensor* forget_gate_bias, + const TfLiteEvalTensor* cell_gate_bias, + const TfLiteEvalTensor* output_gate_bias, + const TfLiteEvalTensor* projection_weights, + const TfLiteEvalTensor* projection_bias, const TfLiteLSTMParams* params, + bool forward_sequence, bool time_major, + const IntegerLstmParameter* integer_lstm_param, int32_t output_state_zp, + TfLiteEvalTensor* output_state, TfLiteEvalTensor* cell_state, + TfLiteEvalTensor* output, int16_t* scratch0, int16_t* scratch1, + int16_t* scratch2, int16_t* scratch3, int8_t* scratch4, int32_t* scratch5) { + TFLITE_DCHECK(input->dims->size >= 2 && input->dims->size <= 3); + const int n_input = input->dims->data[input->dims->size - 1]; + int max_time, n_batch; + if (input->dims->size == 2) { + max_time = 1; + n_batch = input->dims->data[0]; + } else { + max_time = (time_major) ? input->dims->data[0] : input->dims->data[1]; + n_batch = (time_major) ? input->dims->data[1] : input->dims->data[0]; + } + + // n_cell and n_output will be the same size when there is no projection. + const int n_cell = input_to_output_weights->dims->data[0]; + const int n_output = recurrent_to_output_weights->dims->data[1]; + + // Get params for time/batch/sequence. + const int output_batch_leading_dim = + output->dims->data[output->dims->size - 1]; + + if (time_major) { + const int input_step = n_batch * n_input; + const int output_step = n_batch * output_batch_leading_dim; + for (int t = 0; t < max_time; t++) { + const int t_rel = t; + int8_t* output_ptr = + tflite::micro::GetTensorData(output) + t_rel * output_step; + const int8_t* input_ptr = + tflite::micro::GetTensorData(input) + t_rel * input_step; + LstmStepInteger8x8_16( + input_ptr, + input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_input_weights), + integer_lstm_param->effective_input_to_input_scale_a, + integer_lstm_param->effective_input_to_input_scale_b, + input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_forget_weights), + integer_lstm_param->effective_input_to_forget_scale_a, + integer_lstm_param->effective_input_to_forget_scale_b, + input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_cell_weights), + integer_lstm_param->effective_input_to_cell_scale_a, + integer_lstm_param->effective_input_to_cell_scale_b, + input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_output_weights), + integer_lstm_param->effective_input_to_output_scale_a, + integer_lstm_param->effective_input_to_output_scale_b, + recurrent_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_input_weights), + integer_lstm_param->effective_recurrent_to_input_scale_a, + integer_lstm_param->effective_recurrent_to_input_scale_b, + recurrent_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_forget_weights), + integer_lstm_param->effective_recurrent_to_forget_scale_a, + integer_lstm_param->effective_recurrent_to_forget_scale_b, + recurrent_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(recurrent_to_cell_weights), + integer_lstm_param->effective_recurrent_to_cell_scale_a, + integer_lstm_param->effective_recurrent_to_cell_scale_b, + recurrent_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_output_weights), + integer_lstm_param->effective_recurrent_to_output_scale_a, + integer_lstm_param->effective_recurrent_to_output_scale_b, + cell_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_input_weights), + integer_lstm_param->effective_cell_to_input_scale_a, + integer_lstm_param->effective_cell_to_input_scale_b, + cell_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_forget_weights), + integer_lstm_param->effective_cell_to_forget_scale_a, + integer_lstm_param->effective_cell_to_forget_scale_b, + cell_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_output_weights), + integer_lstm_param->effective_cell_to_output_scale_a, + integer_lstm_param->effective_cell_to_output_scale_b, + projection_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_weights), + integer_lstm_param->effective_proj_scale_a, + integer_lstm_param->effective_proj_scale_b, + integer_lstm_param->hidden_zp, + integer_lstm_param->effective_hidden_scale_a, + integer_lstm_param->effective_hidden_scale_b, + input_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_layer_norm_coefficients), + integer_lstm_param->layer_norm_input_scale_a, + integer_lstm_param->layer_norm_input_scale_b, + forget_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + forget_layer_norm_coefficients), + integer_lstm_param->layer_norm_forget_scale_a, + integer_lstm_param->layer_norm_forget_scale_b, + cell_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + cell_layer_norm_coefficients), + integer_lstm_param->layer_norm_cell_scale_a, + integer_lstm_param->layer_norm_cell_scale_b, + output_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + output_layer_norm_coefficients), + integer_lstm_param->layer_norm_output_scale_a, + integer_lstm_param->layer_norm_output_scale_b, + input_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_gate_bias), + forget_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(forget_gate_bias), + cell_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_gate_bias), + output_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(output_gate_bias), + integer_lstm_param->quantized_cell_clip, + integer_lstm_param->quantized_proj_clip, + integer_lstm_param->cell_scale, + integer_lstm_param->input_variance_guard, + integer_lstm_param->forget_variance_guard, + integer_lstm_param->cell_variance_guard, + integer_lstm_param->output_variance_guard, + integer_lstm_param->input_to_forget_effective_bias, + integer_lstm_param->recurrent_to_forget_effective_bias, + integer_lstm_param->input_to_cell_effective_bias, + integer_lstm_param->recurrent_to_cell_effective_bias, + integer_lstm_param->input_to_output_effective_bias, + integer_lstm_param->recurrent_to_output_effective_bias, + integer_lstm_param->input_to_input_effective_bias, + integer_lstm_param->recurrent_to_input_effective_bias, + integer_lstm_param->projection_effective_bias, n_batch, n_cell, + n_input, n_output, tflite::micro::GetTensorData(output_state), + output_state_zp, tflite::micro::GetTensorData(cell_state), + output_ptr, scratch0, scratch1, scratch2, scratch3, scratch4, + scratch5); + } + } else { + for (int b = 0; b < n_batch; b++) { + const int input_step = n_input; + const int output_step = output_batch_leading_dim; + for (int t = 0; t < max_time; t++) { + // If this is the forward_sequence, step forward, otherwise step + // backwards. + const int t_rel = forward_sequence ? t : max_time - t - 1; + const int time_offset = b * max_time + t_rel; + const int8_t* input_ptr = tflite::micro::GetTensorData(input) + + time_offset * input_step; + int8_t* output_ptr = tflite::micro::GetTensorData(output) + + time_offset * output_step; + + // Offset the {output,cell}_state pointers to the right batch. + int8_t* output_state_ptr = + tflite::micro::GetTensorData(output_state) + + b * output_batch_leading_dim; + int16_t* cell_state_ptr = + tflite::micro::GetTensorData(cell_state) + b * n_cell; + + LstmStepInteger8x8_16( + input_ptr, + input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_input_weights), + integer_lstm_param->effective_input_to_input_scale_a, + integer_lstm_param->effective_input_to_input_scale_b, + input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_forget_weights), + integer_lstm_param->effective_input_to_forget_scale_a, + integer_lstm_param->effective_input_to_forget_scale_b, + input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_cell_weights), + integer_lstm_param->effective_input_to_cell_scale_a, + integer_lstm_param->effective_input_to_cell_scale_b, + input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_output_weights), + integer_lstm_param->effective_input_to_output_scale_a, + integer_lstm_param->effective_input_to_output_scale_b, + recurrent_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_input_weights), + integer_lstm_param->effective_recurrent_to_input_scale_a, + integer_lstm_param->effective_recurrent_to_input_scale_b, + recurrent_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_forget_weights), + integer_lstm_param->effective_recurrent_to_forget_scale_a, + integer_lstm_param->effective_recurrent_to_forget_scale_b, + recurrent_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_cell_weights), + integer_lstm_param->effective_recurrent_to_cell_scale_a, + integer_lstm_param->effective_recurrent_to_cell_scale_b, + recurrent_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData( + recurrent_to_output_weights), + integer_lstm_param->effective_recurrent_to_output_scale_a, + integer_lstm_param->effective_recurrent_to_output_scale_b, + cell_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_input_weights), + integer_lstm_param->effective_cell_to_input_scale_a, + integer_lstm_param->effective_cell_to_input_scale_b, + cell_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_forget_weights), + integer_lstm_param->effective_cell_to_forget_scale_a, + integer_lstm_param->effective_cell_to_forget_scale_b, + cell_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_output_weights), + integer_lstm_param->effective_cell_to_output_scale_a, + integer_lstm_param->effective_cell_to_output_scale_b, + projection_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_weights), + integer_lstm_param->effective_proj_scale_a, + integer_lstm_param->effective_proj_scale_b, + integer_lstm_param->hidden_zp, + integer_lstm_param->effective_hidden_scale_a, + integer_lstm_param->effective_hidden_scale_b, + input_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_layer_norm_coefficients), + integer_lstm_param->layer_norm_input_scale_a, + integer_lstm_param->layer_norm_input_scale_b, + forget_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + forget_layer_norm_coefficients), + integer_lstm_param->layer_norm_forget_scale_a, + integer_lstm_param->layer_norm_forget_scale_b, + cell_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + cell_layer_norm_coefficients), + integer_lstm_param->layer_norm_cell_scale_a, + integer_lstm_param->layer_norm_cell_scale_b, + output_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + output_layer_norm_coefficients), + integer_lstm_param->layer_norm_output_scale_a, + integer_lstm_param->layer_norm_output_scale_b, + input_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_gate_bias), + forget_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(forget_gate_bias), + cell_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_gate_bias), + output_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(output_gate_bias), + integer_lstm_param->quantized_cell_clip, + integer_lstm_param->quantized_proj_clip, + integer_lstm_param->cell_scale, + integer_lstm_param->input_variance_guard, + integer_lstm_param->forget_variance_guard, + integer_lstm_param->cell_variance_guard, + integer_lstm_param->output_variance_guard, + integer_lstm_param->input_to_forget_effective_bias, + integer_lstm_param->recurrent_to_forget_effective_bias, + integer_lstm_param->input_to_cell_effective_bias, + integer_lstm_param->recurrent_to_cell_effective_bias, + integer_lstm_param->input_to_output_effective_bias, + integer_lstm_param->recurrent_to_output_effective_bias, + integer_lstm_param->input_to_input_effective_bias, + integer_lstm_param->recurrent_to_input_effective_bias, + integer_lstm_param->projection_effective_bias, /*n_batch=*/1, + n_cell, n_input, n_output, output_state_ptr, output_state_zp, + cell_state_ptr, output_ptr, scratch0, scratch1, scratch2, scratch3, + scratch4, scratch5); + } + } + } + + return kTfLiteOk; +} + +TfLiteStatus EvalInteger8x8_8Lstm( + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* input_to_input_weights, + const TfLiteEvalTensor* input_to_forget_weights, + const TfLiteEvalTensor* input_to_cell_weights, + const TfLiteEvalTensor* input_to_output_weights, + const TfLiteEvalTensor* recurrent_to_input_weights, + const TfLiteEvalTensor* recurrent_to_forget_weights, + const TfLiteEvalTensor* recurrent_to_cell_weights, + const TfLiteEvalTensor* recurrent_to_output_weights, + const TfLiteEvalTensor* cell_to_input_weights, + const TfLiteEvalTensor* cell_to_forget_weights, + const TfLiteEvalTensor* cell_to_output_weights, + const TfLiteEvalTensor* input_layer_norm_coefficients, + const TfLiteEvalTensor* forget_layer_norm_coefficients, + const TfLiteEvalTensor* cell_layer_norm_coefficients, + const TfLiteEvalTensor* output_layer_norm_coefficients, + const TfLiteEvalTensor* input_gate_bias, + const TfLiteEvalTensor* forget_gate_bias, + const TfLiteEvalTensor* cell_gate_bias, + const TfLiteEvalTensor* output_gate_bias, + const TfLiteEvalTensor* projection_weights, + const TfLiteEvalTensor* projection_bias, const TfLiteLSTMParams* params, + TfLiteEvalTensor* output_state, TfLiteEvalTensor* cell_state, + TfLiteEvalTensor* output, const IntegerLstmParameter* integer_lstm_param, + int32_t input_zp, int32_t output_state_zp, int8_t* scratch0, + int8_t* scratch1, int16_t* scratch2, int16_t* scratch3, int16_t* scratch4, + int16_t* scratch5, int16_t* scratch6, int16_t* scratch7) { + TFLITE_DCHECK(input->dims->size >= 2 && input->dims->size <= 3); + const int n_input = input->dims->data[input->dims->size - 1]; + int max_time, n_batch; + if (input->dims->size == 2) { + max_time = 1; + n_batch = input->dims->data[0]; + } else { + max_time = input->dims->data[0]; + n_batch = input->dims->data[1]; + } + + // n_cell and n_output will be the same size when there is no projection. + const int n_cell = input_to_output_weights->dims->data[0]; + const int n_output = recurrent_to_output_weights->dims->data[1]; + + // Get params for time/batch/sequence. + const int output_batch_leading_dim = + output->dims->data[output->dims->size - 1]; + const int input_step = n_batch * n_input; + const int output_step = n_batch * output_batch_leading_dim; + + for (int t = 0; t < max_time; t++) { + const int t_rel = t; + int8_t* output_ptr = + tflite::micro::GetTensorData(output) + t_rel * output_step; + // Input can be int8 asymmetric or int16 symmetric. + const int8_t* input_ptr = + tflite::micro::GetTensorData(input) + t_rel * input_step; + LstmStepInteger8x8_8( + input_ptr, input_zp, + + input_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_input_weights), + integer_lstm_param->effective_input_to_input_scale_a, + integer_lstm_param->effective_input_to_input_scale_b, + + input_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_forget_weights), + integer_lstm_param->effective_input_to_forget_scale_a, + integer_lstm_param->effective_input_to_forget_scale_b, + + input_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_cell_weights), + integer_lstm_param->effective_input_to_cell_scale_a, + integer_lstm_param->effective_input_to_cell_scale_b, + + input_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_to_output_weights), + integer_lstm_param->effective_input_to_output_scale_a, + integer_lstm_param->effective_input_to_output_scale_b, + + recurrent_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(recurrent_to_input_weights), + integer_lstm_param->effective_recurrent_to_input_scale_a, + integer_lstm_param->effective_recurrent_to_input_scale_b, + + recurrent_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(recurrent_to_forget_weights), + integer_lstm_param->effective_recurrent_to_forget_scale_a, + integer_lstm_param->effective_recurrent_to_forget_scale_b, + + recurrent_to_cell_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(recurrent_to_cell_weights), + integer_lstm_param->effective_recurrent_to_cell_scale_a, + integer_lstm_param->effective_recurrent_to_cell_scale_b, + + recurrent_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(recurrent_to_output_weights), + integer_lstm_param->effective_recurrent_to_output_scale_a, + integer_lstm_param->effective_recurrent_to_output_scale_b, + + cell_to_input_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_input_weights), + integer_lstm_param->effective_cell_to_input_scale_a, + integer_lstm_param->effective_cell_to_input_scale_b, + + cell_to_forget_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_forget_weights), + integer_lstm_param->effective_cell_to_forget_scale_a, + integer_lstm_param->effective_cell_to_forget_scale_b, + + cell_to_output_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_to_output_weights), + integer_lstm_param->effective_cell_to_output_scale_a, + integer_lstm_param->effective_cell_to_output_scale_b, + + projection_weights == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_weights), + integer_lstm_param->effective_proj_scale_a, + integer_lstm_param->effective_proj_scale_b, + + input_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + input_layer_norm_coefficients), + integer_lstm_param->layer_norm_input_scale_a, + integer_lstm_param->layer_norm_input_scale_b, + + forget_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + forget_layer_norm_coefficients), + integer_lstm_param->layer_norm_forget_scale_a, + integer_lstm_param->layer_norm_forget_scale_b, + + cell_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + cell_layer_norm_coefficients), + integer_lstm_param->layer_norm_cell_scale_a, + integer_lstm_param->layer_norm_cell_scale_b, + + output_layer_norm_coefficients == nullptr + ? nullptr + : tflite::micro::GetTensorData( + output_layer_norm_coefficients), + integer_lstm_param->layer_norm_output_scale_a, + integer_lstm_param->layer_norm_output_scale_b, + + input_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(input_gate_bias), + forget_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(forget_gate_bias), + cell_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(cell_gate_bias), + output_gate_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(output_gate_bias), + projection_bias == nullptr + ? nullptr + : tflite::micro::GetTensorData(projection_bias), + + params, integer_lstm_param->intermediate_scale_a, + integer_lstm_param->intermediate_scale_b, + integer_lstm_param->intermediate_zp, + integer_lstm_param->quantized_cell_clip, + integer_lstm_param->quantized_proj_clip, n_batch, n_cell, n_input, + n_output, output_batch_leading_dim, + tflite::micro::GetTensorData(output_state), output_state_zp, + tflite::micro::GetTensorData(cell_state), output_ptr, scratch0, + scratch1, scratch2, scratch3, scratch4, scratch5, scratch6, scratch7); + } + + return kTfLiteOk; +} + +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_eval.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_eval.h new file mode 100644 index 00000000..218b4938 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_eval.h @@ -0,0 +1,250 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_LSTM_EVAL_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_LSTM_EVAL_H_ + +#include +#include + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" + +namespace tflite { + +// Pamameters for integer LSTM. +// Consider split this into two Integer Parameters if more fields are added. +struct IntegerLstmParameter { + int32_t effective_input_to_input_scale_a; + int32_t effective_input_to_input_scale_b; + int32_t effective_recurrent_to_input_scale_a; + int32_t effective_recurrent_to_input_scale_b; + int32_t effective_cell_to_input_scale_a; + int32_t effective_cell_to_input_scale_b; + int32_t effective_input_to_forget_scale_a; + int32_t effective_input_to_forget_scale_b; + int32_t effective_recurrent_to_forget_scale_a; + int32_t effective_recurrent_to_forget_scale_b; + int32_t effective_cell_to_forget_scale_a; + int32_t effective_cell_to_forget_scale_b; + int32_t effective_input_to_cell_scale_a; + int32_t effective_input_to_cell_scale_b; + int32_t effective_recurrent_to_cell_scale_a; + int32_t effective_recurrent_to_cell_scale_b; + int32_t effective_input_to_output_scale_a; + int32_t effective_input_to_output_scale_b; + int32_t effective_recurrent_to_output_scale_a; + int32_t effective_recurrent_to_output_scale_b; + int32_t effective_cell_to_output_scale_a; + int32_t effective_cell_to_output_scale_b; + int32_t effective_proj_scale_a; + int32_t effective_proj_scale_b; + int32_t effective_hidden_scale_a; + int32_t effective_hidden_scale_b; + int32_t layer_norm_input_scale_a; + int32_t layer_norm_input_scale_b; + int32_t layer_norm_forget_scale_a; + int32_t layer_norm_forget_scale_b; + int32_t layer_norm_cell_scale_a; + int32_t layer_norm_cell_scale_b; + int32_t layer_norm_output_scale_a; + int32_t layer_norm_output_scale_b; + // Quantized clip value for cell and projection. Zero value means no clipping. + int16_t quantized_cell_clip; + int8_t quantized_proj_clip; + int32_t hidden_zp; + int32_t cell_scale; + + int32_t input_variance_guard; + int32_t forget_variance_guard; + int32_t cell_variance_guard; + int32_t output_variance_guard; + + // Pre-calculate bias + zero_point * weight. + int32_t* input_to_forget_effective_bias; + int32_t* recurrent_to_forget_effective_bias; + int32_t* input_to_cell_effective_bias; + int32_t* recurrent_to_cell_effective_bias; + int32_t* input_to_output_effective_bias; + int32_t* recurrent_to_output_effective_bias; + int32_t* input_to_input_effective_bias; + int32_t* recurrent_to_input_effective_bias; + int32_t* projection_effective_bias; + + // Scale and zero point for intermediate tensors. + // Used only in the 8x8_8 case. + int32_t intermediate_scale_a[8]; + int32_t intermediate_scale_b[8]; + int32_t intermediate_zp[12]; +}; + +// Scales for hybrid op with integer inputs and float weights +struct HybridLstmScales { + float input_to_input_weights_scale; + float input_to_forget_weights_scale; + float input_to_cell_weights_scale; + float input_to_output_weights_scale; + float aux_input_to_input_weights_scale; + float aux_input_to_forget_weights_scale; + float aux_input_to_cell_weights_scale; + float aux_input_to_output_weights_scale; + float recurrent_to_input_weights_scale; + float recurrent_to_forget_weights_scale; + float recurrent_to_cell_weights_scale; + float recurrent_to_output_weights_scale; + float cell_to_input_weights_scale; + float cell_to_forget_weights_scale; + float cell_to_output_weights_scale; + float projection_weights_scale; +}; + +TfLiteStatus EvalFloatLstm( + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* input_to_input_weights, + const TfLiteEvalTensor* input_to_forget_weights, + const TfLiteEvalTensor* input_to_cell_weights, + const TfLiteEvalTensor* input_to_output_weights, + const TfLiteEvalTensor* recurrent_to_input_weights, + const TfLiteEvalTensor* recurrent_to_forget_weights, + const TfLiteEvalTensor* recurrent_to_cell_weights, + const TfLiteEvalTensor* recurrent_to_output_weights, + const TfLiteEvalTensor* cell_to_input_weights, + const TfLiteEvalTensor* cell_to_forget_weights, + const TfLiteEvalTensor* cell_to_output_weights, + const TfLiteEvalTensor* input_layer_norm_coefficients, + const TfLiteEvalTensor* forget_layer_norm_coefficients, + const TfLiteEvalTensor* cell_layer_norm_coefficients, + const TfLiteEvalTensor* output_layer_norm_coefficients, + const TfLiteEvalTensor* aux_input, + const TfLiteEvalTensor* aux_input_to_input_weights, + const TfLiteEvalTensor* aux_input_to_forget_weights, + const TfLiteEvalTensor* aux_input_to_cell_weights, + const TfLiteEvalTensor* aux_input_to_output_weights, + const TfLiteEvalTensor* input_gate_bias, + const TfLiteEvalTensor* forget_gate_bias, + const TfLiteEvalTensor* cell_gate_bias, + const TfLiteEvalTensor* output_gate_bias, + const TfLiteEvalTensor* projection_weights, + const TfLiteEvalTensor* projection_bias, const TfLiteLSTMParams* params, + bool forward_sequence, bool time_major, int output_offset, + float* scratch_buffer, TfLiteEvalTensor* output_state, + TfLiteEvalTensor* cell_state, TfLiteEvalTensor* output); + +TfLiteStatus EvalHybridLstm( + const HybridLstmScales* hybrid_lstm_scales, const TfLiteEvalTensor* input, + const TfLiteEvalTensor* input_to_input_weights, + const TfLiteEvalTensor* input_to_input_weights_ledger, + const TfLiteEvalTensor* input_to_forget_weights, + const TfLiteEvalTensor* input_to_forget_weights_ledger, + const TfLiteEvalTensor* input_to_cell_weights, + const TfLiteEvalTensor* input_to_cell_weights_ledger, + const TfLiteEvalTensor* input_to_output_weights, + const TfLiteEvalTensor* input_to_output_weights_ledger, + const TfLiteEvalTensor* recurrent_to_input_weights, + const TfLiteEvalTensor* recurrent_to_input_weights_ledger, + const TfLiteEvalTensor* recurrent_to_forget_weights, + const TfLiteEvalTensor* recurrent_to_forget_weights_ledger, + const TfLiteEvalTensor* recurrent_to_cell_weights, + const TfLiteEvalTensor* recurrent_to_cell_weights_ledger, + const TfLiteEvalTensor* recurrent_to_output_weights, + const TfLiteEvalTensor* recurrent_to_output_weights_ledger, + const TfLiteEvalTensor* cell_to_input_weights, + const TfLiteEvalTensor* cell_to_forget_weights, + const TfLiteEvalTensor* cell_to_output_weights, + const TfLiteEvalTensor* input_layer_norm_coefficients, + const TfLiteEvalTensor* forget_layer_norm_coefficients, + const TfLiteEvalTensor* cell_layer_norm_coefficients, + const TfLiteEvalTensor* output_layer_norm_coefficients, + const TfLiteEvalTensor* aux_input, + const TfLiteEvalTensor* aux_input_to_input_weights, + const TfLiteEvalTensor* aux_input_to_forget_weights, + const TfLiteEvalTensor* aux_input_to_cell_weights, + const TfLiteEvalTensor* aux_input_to_output_weights, + const TfLiteEvalTensor* input_gate_bias, + const TfLiteEvalTensor* forget_gate_bias, + const TfLiteEvalTensor* cell_gate_bias, + const TfLiteEvalTensor* output_gate_bias, + const TfLiteEvalTensor* projection_weights, + const TfLiteEvalTensor* projection_weights_ledger, + const TfLiteEvalTensor* projection_bias, const TfLiteLSTMParams* params, + bool forward_sequence, bool time_major, int output_offset, + float* scratch_buffer, float* input_sf, float* aux_input_sf, + float* output_state_sf, float* prod_scaling_factors, + float* recovered_cell_weights, int8_t* input_quantized, + int8_t* aux_input_quantized, int8_t* output_state_quantized, + int8_t* cell_state_quantized, float* scales, TfLiteEvalTensor* output_state, + TfLiteEvalTensor* cell_state, int32_t* output_scratch_buffer, + TfLiteEvalTensor* output, int32_t* input_zp, int32_t* aux_input_zp, + int32_t* output_state_zp, int32_t* row_sums, int row_sums_size, + bool* compute_row_sums); + +TfLiteStatus EvalInteger8x8_16Lstm( + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* input_to_input_weights, + const TfLiteEvalTensor* input_to_forget_weights, + const TfLiteEvalTensor* input_to_cell_weights, + const TfLiteEvalTensor* input_to_output_weights, + const TfLiteEvalTensor* recurrent_to_input_weights, + const TfLiteEvalTensor* recurrent_to_forget_weights, + const TfLiteEvalTensor* recurrent_to_cell_weights, + const TfLiteEvalTensor* recurrent_to_output_weights, + const TfLiteEvalTensor* cell_to_input_weights, + const TfLiteEvalTensor* cell_to_forget_weights, + const TfLiteEvalTensor* cell_to_output_weights, + const TfLiteEvalTensor* input_layer_norm_coefficients, + const TfLiteEvalTensor* forget_layer_norm_coefficients, + const TfLiteEvalTensor* cell_layer_norm_coefficients, + const TfLiteEvalTensor* output_layer_norm_coefficients, + const TfLiteEvalTensor* input_gate_bias, + const TfLiteEvalTensor* forget_gate_bias, + const TfLiteEvalTensor* cell_gate_bias, + const TfLiteEvalTensor* output_gate_bias, + const TfLiteEvalTensor* projection_weights, + const TfLiteEvalTensor* projection_bias, const TfLiteLSTMParams* params, + bool forward_sequence, bool time_major, + const IntegerLstmParameter* integer_lstm_param, int32_t output_state_zp, + TfLiteEvalTensor* output_state, TfLiteEvalTensor* cell_state, + TfLiteEvalTensor* output, int16_t* scratch0, int16_t* scratch1, + int16_t* scratch2, int16_t* scratch3, int8_t* scratch4, int32_t* scratch5); + +TfLiteStatus EvalInteger8x8_8Lstm( + const TfLiteEvalTensor* input, + const TfLiteEvalTensor* input_to_input_weights, + const TfLiteEvalTensor* input_to_forget_weights, + const TfLiteEvalTensor* input_to_cell_weights, + const TfLiteEvalTensor* input_to_output_weights, + const TfLiteEvalTensor* recurrent_to_input_weights, + const TfLiteEvalTensor* recurrent_to_forget_weights, + const TfLiteEvalTensor* recurrent_to_cell_weights, + const TfLiteEvalTensor* recurrent_to_output_weights, + const TfLiteEvalTensor* cell_to_input_weights, + const TfLiteEvalTensor* cell_to_forget_weights, + const TfLiteEvalTensor* cell_to_output_weights, + const TfLiteEvalTensor* input_layer_norm_coefficients, + const TfLiteEvalTensor* forget_layer_norm_coefficients, + const TfLiteEvalTensor* cell_layer_norm_coefficients, + const TfLiteEvalTensor* output_layer_norm_coefficients, + const TfLiteEvalTensor* input_gate_bias, + const TfLiteEvalTensor* forget_gate_bias, + const TfLiteEvalTensor* cell_gate_bias, + const TfLiteEvalTensor* output_gate_bias, + const TfLiteEvalTensor* projection_weights, + const TfLiteEvalTensor* projection_bias, const TfLiteLSTMParams* params, + TfLiteEvalTensor* output_state, TfLiteEvalTensor* cell_state, + TfLiteEvalTensor* output, const IntegerLstmParameter* integer_lstm_param, + int8_t* scratch0, int8_t* scratch1, int16_t* scratch2, int16_t* scratch3, + int16_t* scratch4, int16_t* scratch5, int16_t* scratch6, int16_t* scratch7); + +} // namespace tflite +#endif // TENSORFLOW_LITE_MICRO_KERNELS_LSTM_EVAL_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_shared.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_shared.h new file mode 100644 index 00000000..ee34b848 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/lstm_shared.h @@ -0,0 +1,67 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_LSTM_SHARED_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_LSTM_SHARED_H_ + +namespace tflite { + +// Input Tensors of size {n_batch, n_input} +constexpr int kLstmInputTensor = 0; + +// Input weight tensors of size: {n_cell, n_input} +constexpr int kLstmInputToInputWeightsTensor = 1; // Optional +constexpr int kLstmInputToForgetWeightsTensor = 2; +constexpr int kLstmInputToCellWeightsTensor = 3; +constexpr int kLstmInputToOutputWeightsTensor = 4; + +// Recurrent weight tensors of size {n_cell, n_output} +constexpr int kLstmRecurrentToInputWeightsTensor = 5; // Optional +constexpr int kLstmRecurrentToForgetWeightsTensor = 6; +constexpr int kLstmRecurrentToCellWeightsTensor = 7; +constexpr int kLstmRecurrentToOutputWeightsTensor = 8; + +// Peephole weights tensors of size {n_cell}, representing a diagonal matrix. +constexpr int kLstmCellToInputWeightsTensor = 9; // Optional +constexpr int kLstmCellToForgetWeightsTensor = 10; // Optional +constexpr int kLstmCellToOutputWeightsTensor = 11; // Optional + +// Gates bias tensors of size {n_cell} +constexpr int kLstmInputGateBiasTensor = 12; // Optional +constexpr int kLstmForgetGateBiasTensor = 13; +constexpr int kLstmCellGateBiasTensor = 14; +constexpr int kLstmOutputGateBiasTensor = 15; + +// Projection weight tensor of size {n_output, n_cell} +constexpr int kLstmProjectionWeightsTensor = 16; // Optional +// Projection bias tensor of size {n_output} +constexpr int kLstmProjectionBiasTensor = 17; // Optional + +// These state tensors are defined as variable tensors, and will be modified by +// this op. +constexpr int kLstmOutputStateTensor = 18; +constexpr int kLstmCellStateTensor = 19; + +// Layer norm coefficient tensors of size {n_cell}, representing a diagonal +// matrix. +constexpr int kLstmInputLayerNormCoefficientsTensor = 20; // Optional +constexpr int kLstmForgetLayerNormCoefficientsTensor = 21; // Optional +constexpr int kLstmCellLayerNormCoefficientsTensor = 22; // Optional +constexpr int kLstmOutputLayerNormCoefficientsTensor = 23; // Optional + +// Output tensors. +constexpr int kLstmOutputTensor = 0; + +} // namespace tflite +#endif // TENSORFLOW_LITE_MICRO_KERNELS_LSTM_SHARED_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/maximum_minimum.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/maximum_minimum.cc index a6d358fb..7964f1e6 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/maximum_minimum.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/maximum_minimum.cc @@ -115,29 +115,17 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace maximum_minimum TfLiteRegistration Register_MAXIMUM() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/ - maximum_minimum::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + nullptr, nullptr, + maximum_minimum::Eval); } TfLiteRegistration Register_MINIMUM() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/ - maximum_minimum::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + nullptr, nullptr, + maximum_minimum::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_ops.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_ops.h index 17064fee..c4dec92d 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_ops.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_ops.h @@ -76,11 +76,14 @@ TfLiteRegistration Register_SHAPE(); TfLiteRegistration Register_SLICE(); TfLiteRegistration Register_SPACE_TO_BATCH_ND(); TfLiteRegistration Register_SPACE_TO_DEPTH(); +TfLiteRegistration Register_SQUARED_DIFFERENCE(); TfLiteRegistration Register_SQUEEZE(); TfLiteRegistration Register_SUB(); TfLiteRegistration Register_SVDF(); TfLiteRegistration Register_TRANSPOSE(); TfLiteRegistration Register_TRANSPOSE_CONV(); +// TODO(b/230666079): resolve conflict with xtensa implementation +TfLiteRegistration Register_UNIDIRECTIONAL_SEQUENCE_LSTM(); TfLiteRegistration Register_VAR_HANDLE(); TfLiteRegistration Register_WHILE(); TfLiteRegistration Register_ZEROS_LIKE(); @@ -103,14 +106,12 @@ TfLiteRegistration Register_LESS_EQUAL(); TfLiteRegistration Register_LOG(); TfLiteRegistration Register_LOGICAL_NOT(); TfLiteRegistration Register_MAXIMUM(); -TfLiteRegistration Register_MEAN(); TfLiteRegistration Register_MINIMUM(); TfLiteRegistration Register_NEG(); TfLiteRegistration Register_NOT_EQUAL(); TfLiteRegistration Register_PACK(); TfLiteRegistration Register_PAD(); TfLiteRegistration Register_PADV2(); -TfLiteRegistration Register_REDUCE_MAX(); TfLiteRegistration Register_RESHAPE(); TfLiteRegistration Register_RESIZE_NEAREST_NEIGHBOR(); TfLiteRegistration Register_ROUND(); @@ -121,7 +122,6 @@ TfLiteRegistration Register_SPLIT_V(); TfLiteRegistration Register_SQRT(); TfLiteRegistration Register_SQUARE(); TfLiteRegistration Register_STRIDED_SLICE(); -TfLiteRegistration Register_UNIDIRECTIONAL_SEQUENCE_LSTM(); TfLiteRegistration Register_UNPACK(); TfLiteRegistration Register_L2_NORMALIZATION(); TfLiteRegistration Register_TANH(); diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_tensor_utils.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_tensor_utils.cc new file mode 100644 index 00000000..88b097c7 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_tensor_utils.cc @@ -0,0 +1,809 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/micro/kernels/micro_tensor_utils.h" + +#include +#include +#include +#include +#include +#include + +#include "fixedpoint/fixedpoint.h" // from @gemmlowp +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/kernels/internal/cppmath.h" +#include "tensorflow/lite/kernels/op_macros.h" + +namespace tflite { +namespace micro_tensor_utils { + +namespace { +const int32_t kInt16Max = std::numeric_limits::max(); +const int32_t kInt16Min = std::numeric_limits::min(); +} // namespace + +void PortableSymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, float* min_value, + float* max_value, float* scaling_factor) { + auto minmax = std::minmax_element(values, values + size); + *min_value = *minmax.first; + *max_value = *minmax.second; + + PortableSymmetricQuantizeFloats(values, size, quantized_values, *min_value, + *max_value, scaling_factor); +} + +void PortableSymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, float min_value, + float max_value, float* scaling_factor) { + const int32_t kScale = 127; + const float range = std::max(std::abs(min_value), std::abs(max_value)); + if (range == 0) { + memset(quantized_values, 0, size * sizeof(int8_t)); + *scaling_factor = 1; + return; + } + *scaling_factor = range / kScale; + const float scaling_factor_inv = kScale / range; + for (int i = 0; i < size; ++i) { + const int32_t quantized_value = + static_cast(TfLiteRound(values[i] * scaling_factor_inv)); + // Clamp: just in case some odd numeric offset. + quantized_values[i] = static_cast( + std::min(kScale, std::max(-kScale, quantized_value))); + } +} + +void PortableAsymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, + float* scaling_factor, int32_t* offset) { + const int32_t kMinScale = -128; + const int32_t kMaxScale = 127; + const double qmin_double = kMinScale; + const double qmax_double = kMaxScale; + const auto minmax = std::minmax_element(values, values + size); + const double rmin = static_cast(std::min(0.0f, *minmax.first)); + const double rmax = static_cast(std::max(0.0f, *minmax.second)); + if (rmin == rmax) { + memset(quantized_values, 0, size * sizeof(int8_t)); + *scaling_factor = 1; + *offset = 0; + return; + } else { + double scale = (rmax - rmin) / (qmax_double - qmin_double); + const double zero_point_from_min = qmin_double - rmin / scale; + const double zero_point_from_max = qmax_double - rmax / scale; + const double zero_point_from_min_error = + std::abs(qmin_double) + std::abs(rmin / scale); + const double zero_point_from_max_error = + std::abs(qmax_double) + std::abs(rmax / scale); + const double zero_point_double = + zero_point_from_min_error < zero_point_from_max_error + ? zero_point_from_min + : zero_point_from_max; + int8_t nudged_zero_point = 0; + if (zero_point_double <= qmin_double) { + nudged_zero_point = kMinScale; + } else if (zero_point_double >= qmax_double) { + nudged_zero_point = kMaxScale; + } else { + nudged_zero_point = static_cast(round(zero_point_double)); + } + *scaling_factor = scale; + *offset = nudged_zero_point; + } + const float scaling_factor_inv = 1.0f / *scaling_factor; + for (int i = 0; i < size; ++i) { + const int32_t quantized_value = static_cast( + TfLiteRound(*offset + values[i] * scaling_factor_inv)); + quantized_values[i] = + std::min(kMaxScale, std::max(kMinScale, quantized_value)); + } +} + +void PortableMatrixBatchVectorMultiplyAccumulate(const float* matrix, + int m_rows, int m_cols, + const float* vector, + int n_batch, float* result) { + float* result_in_batch = result; + for (int b = 0; b < n_batch; b++) { + const float* matrix_ptr = matrix; + for (int r = 0; r < m_rows; r++) { + float dot_prod = 0.0f; + const float* vector_in_batch = vector + b * m_cols; + for (int c = 0; c < m_cols; c++) { + dot_prod += *matrix_ptr++ * *vector_in_batch++; + } + *result_in_batch += dot_prod; + ++result_in_batch; + } + } +} + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vectors, const float* scaling_factors, + int n_batch, float* __restrict__ result) { + for (int batch = 0; batch < n_batch; ++batch, vectors += m_cols) { + const float batch_scaling_factor = scaling_factors[batch]; + // Get the address of the first row. + const int8_t* row_ptr = matrix; + for (int row = 0; row < m_rows; ++row) { + // Initialize the dot product sum for the row to 0. + int32_t dotprod = 0; + // TODO(b/230666277): remove this +#if defined(__GNUC__) + // Prefetch the row to cache. + __builtin_prefetch(row_ptr, 0 /* prefetch for read */, + 3 /* temporal locality */); +#endif + for (int col = 0; col < m_cols; ++col, ++row_ptr) { + dotprod += (*row_ptr) * (vectors[col]); + } // for col + *result += dotprod * batch_scaling_factor; + ++result; + } // for row + } // for batch +} + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vectors, const float* scaling_factors, + int n_batch, float* __restrict__ result, const float* per_channel_scale, + const int32_t* input_offset, int32_t* scratch, int32_t* row_sums, + bool* compute_row_sums, CpuBackendContext* context) { + if (input_offset == nullptr) { + PortableMatrixBatchVectorMultiplyAccumulate( + matrix, m_rows, m_cols, vectors, scaling_factors, n_batch, result); + return; + } + if (!compute_row_sums || *compute_row_sums) { + PortableReductionSumVector(matrix, row_sums, m_rows, m_cols); + if (compute_row_sums) { + *compute_row_sums = false; + } + } + + for (int batch = 0; batch < n_batch; ++batch, vectors += m_cols) { + const float batch_scaling_factor = scaling_factors[batch]; + const int32_t batch_offset = input_offset[batch]; + const int8_t* row_ptr = matrix; + for (int row = 0; row < m_rows; ++row) { + int32_t dotprod = 0; + float scale = batch_scaling_factor; + if (per_channel_scale) { + scale *= per_channel_scale[row]; + } +#if defined(__GNUC__) + // Prefetch the row to cache. + __builtin_prefetch(row_ptr, 0 /* prefetch for read */, + 3 /* temporal locality */); +#endif + for (int col = 0; col < m_cols; ++col, ++row_ptr) { + dotprod += (*row_ptr) * vectors[col]; + } // for col + dotprod -= row_sums[row] * batch_offset; + *result += dotprod * scale; + ++result; + } // for row + } // for batch +} + +void PortableSparseMatrixBatchVectorMultiplyAccumulate1x4( + const float* __restrict__ matrix, const int32_t* __restrict__ segments, + const int32_t* __restrict__ indices, int m_rows, int m_cols, + const float* __restrict__ vector, int n_batch, float* __restrict__ result) { + const int kBlockSize = 4; + TFLITE_DCHECK_EQ(m_cols % kBlockSize, 0); + for (int batch = 0; batch < n_batch; batch++) { + const float* matrix_ptr = matrix; + for (int row = 0; row < m_rows; row++) { + float dot_prod = 0.0f; + const float* vector_in_batch = vector + batch * m_cols; + for (int i = segments[row]; i < segments[row + 1]; i++) { + const int block_start_index = indices[i] * kBlockSize; + const float* vector_block_in_batch_ptr = + vector_in_batch + block_start_index; + for (int c = 0; c < kBlockSize; c++) { + dot_prod += *matrix_ptr++ * *vector_block_in_batch_ptr++; + } + } + result[batch * m_rows + row] += dot_prod; + } + } +} + +void PortableSparseMatrixBatchVectorMultiplyAccumulate1x16( + const int8_t* __restrict__ matrix, const int32_t* __restrict__ segments, + const int32_t* __restrict__ indices, int m_rows, int m_cols, + const int8_t* __restrict__ vector, const int32_t* __restrict__ bias_vector, + int n_batch, const int32_t input_offset, const int32_t output_multiplier, + const int32_t output_shift, const int32_t output_offset, + const int32_t output_activation_min, const int32_t output_activation_max, + int8_t* __restrict__ result) { + const int kBlockSize = 16; + TFLITE_DCHECK_EQ(m_cols % kBlockSize, 0); + for (int batch = 0; batch < n_batch; ++batch) { + const int8_t* matrix_ptr = matrix; + for (int row = 0; row < m_rows; ++row) { + int32_t dot_prod = 0; + const int8_t* vector_in_batch = vector + batch * m_cols; + for (int i = segments[row]; i < segments[row + 1]; ++i) { + const int block_start_index = indices[i] * kBlockSize; + const int8_t* vector_block_in_batch_ptr = + vector_in_batch + block_start_index; + for (int c = 0; c < kBlockSize; c++) { + dot_prod += *matrix_ptr * *vector_block_in_batch_ptr++; + dot_prod += *matrix_ptr++ * input_offset; + } + } + const int32_t bias_value = bias_vector != nullptr ? bias_vector[row] : 0; + dot_prod = MultiplyByQuantizedMultiplier(dot_prod + bias_value, + output_multiplier, output_shift); + dot_prod += output_offset; + result[batch * m_rows + row] = + static_cast(ActivationFunctionWithMinMax( + dot_prod, output_activation_min, output_activation_max)); + } + } +} + +void PortableSparseMatrixBatchVectorMultiplyAccumulate( + const float* __restrict__ matrix, const uint8_t* __restrict__ ledger, + int m_rows, int m_cols, const float* __restrict__ vector, int n_batch, + float* __restrict__ result) { + const int kBlockSize = 16; + TFLITE_DCHECK_EQ( // NOLINT + m_cols % kBlockSize, 0); + for (int batch = 0; batch < n_batch; batch++) { + const float* matrix_ptr = matrix; + const uint8_t* ledger_ptr = ledger; + for (int row = 0; row < m_rows; row++) { + float dot_prod = 0.0f; + int num_nonzero_blocks = *ledger_ptr++; + if (num_nonzero_blocks > 0) { + const float* vector_in_batch = vector + batch * m_cols; + for (int i = 0; i < num_nonzero_blocks; i++) { + const int block_start_index = *ledger_ptr++ * kBlockSize; + const float* vector_block_in_batch_ptr = + vector_in_batch + block_start_index; + for (int c = 0; c < kBlockSize; c++) { + dot_prod += *matrix_ptr++ * *vector_block_in_batch_ptr++; + } + } + } + result[batch * m_rows + row] += dot_prod; + } + } +} + +void PortableSparseMatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows, + const int m_cols, const int8_t* __restrict__ vectors, + const float* scaling_factors, int n_batch, float* __restrict__ result) { + static const int kBlockSize = 16; + TFLITE_DCHECK_EQ( // NOLINT + m_cols % kBlockSize, 0); + for (int batch = 0; batch < n_batch; ++batch, vectors += m_cols) { + const float batch_scaling_factor = scaling_factors[batch]; + const uint8_t* ledger_ptr = ledger; + // Get the address of the first row. + const int8_t* row_ptr = matrix; + for (int row = 0; row < m_rows; ++row) { + // Initialize the dot product sum for the row to 0. + int32_t dotprod = 0; +#if defined(__GNUC__) + // Prefetch the row to cache. + __builtin_prefetch(row_ptr, 0 /* prefetch for read */, + 3 /* temporal locality */); +#endif + int num_nonzero_blocks = *ledger_ptr++; + for (int i = 0; i < num_nonzero_blocks; i++) { + const int block_start_index = *ledger_ptr++ * kBlockSize; + const int8_t* vector_block_ptr = vectors + block_start_index; + for (int c = 0; c < kBlockSize; c++) { + dotprod += (*row_ptr++) * (*vector_block_ptr++); + } // for block + } // for num_nonzero_blocks + result[batch * m_rows + row] += dotprod * batch_scaling_factor; + } // for row + } // for batch +} + +template +void PortableMatrixBatchVectorMultiplyAccumulateImpl( + const int8_t* input, const int32_t* bias, + const int8_t* input_to_gate_weights, int32_t multiplier, int32_t shift, + int32_t n_batch, int32_t n_input, int32_t n_output, int32_t output_zp, + T* output) { + const int16_t output_max = std::numeric_limits::max(); + const int16_t output_min = std::numeric_limits::min(); + for (int batch = 0; batch < n_batch; ++batch) { + for (int row = 0; row < n_output; ++row) { + int32_t acc = bias[row]; + for (int col = 0; col < n_input; ++col) { + int8_t input_val = input[batch * n_input + col]; + int8_t weights_val = input_to_gate_weights[row * n_input + col]; + acc += input_val * weights_val; + } + acc = MultiplyByQuantizedMultiplier(acc, multiplier, shift); + acc += output_zp; + acc += output[batch * n_output + row]; + if (acc > output_max) { + acc = output_max; + } + if (acc < output_min) { + acc = output_min; + } + output[batch * n_output + row] = static_cast(acc); + } + } +} + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* input, const int32_t* bias, + const int8_t* input_to_gate_weights, int32_t multiplier, int32_t shift, + int32_t n_batch, int32_t n_input, int32_t n_output, int32_t output_zp, + int32_t* scratch, int16_t* output, CpuBackendContext* context) { + PortableMatrixBatchVectorMultiplyAccumulateImpl( + input, bias, input_to_gate_weights, multiplier, shift, n_batch, n_input, + n_output, output_zp, output); +} + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* input, const int32_t* bias, + const int8_t* input_to_gate_weights, int32_t multiplier, int32_t shift, + int32_t n_batch, int32_t n_input, int32_t n_output, int32_t output_zp, + int32_t* scratch, int8_t* output, CpuBackendContext* context) { + PortableMatrixBatchVectorMultiplyAccumulateImpl( + input, bias, input_to_gate_weights, multiplier, shift, n_batch, n_input, + n_output, output_zp, output); +} + +void PortableMatrixBatchVectorMultiply(const int8_t* input, + int32_t input_zeropoint, + const int8_t* input_to_gate_weights, + int32_t input_to_gate_effective_scale_a, + int32_t input_to_gate_effective_scale_b, + int32_t n_batch, int32_t n_input, + int32_t n_cell, int8_t* gate_output, + int8_t gate_output_zp) { + const int32_t int8_max = std::numeric_limits::max(); + const int32_t int8_min = std::numeric_limits::min(); + for (int batch = 0; batch < n_batch; ++batch) { + for (int row = 0; row < n_cell; ++row) { + int32_t acc = 0; + for (int col = 0; col < n_input; ++col) { + int32_t input_val = input[batch * n_input + col]; + int8_t weights_val = input_to_gate_weights[row * n_input + col]; + acc += (input_val - input_zeropoint) * weights_val; + } + acc = MultiplyByQuantizedMultiplier(acc, input_to_gate_effective_scale_a, + input_to_gate_effective_scale_b); + acc += gate_output_zp; + if (acc > int8_max) { + acc = int8_max; + } + if (acc < int8_min) { + acc = int8_min; + } + gate_output[batch * n_cell + row] = static_cast(acc); + } + } +} + +void PortableMatrixBatchVectorMultiply( + const int16_t* hidden, const int8_t* hidden_to_output_weights, + int32_t proj_effective_scale_a, int32_t proj_effective_scale_b, + const int32_t* gate_bias, int32_t n_batch, int32_t n_hidden, + int32_t n_output, int32_t output_zp, int8_t* proj_output) { + const int16_t int8_max = std::numeric_limits::max(); + const int16_t int8_min = std::numeric_limits::min(); + for (int batch = 0; batch < n_batch; ++batch) { + for (int row = 0; row < n_output; ++row) { + int64_t acc = gate_bias[row]; + for (int col = 0; col < n_hidden; ++col) { + int16_t input_val = hidden[batch * n_hidden + col]; + int8_t weights_val = hidden_to_output_weights[row * n_hidden + col]; + int64_t curr = acc; + acc += input_val * weights_val; + if (input_val * weights_val > 0 && acc < curr) { + acc = std::numeric_limits::max(); + } + if (input_val * weights_val < 0 && acc > curr) { + acc = std::numeric_limits::min(); + } + } + acc = MultiplyByQuantizedMultiplier(acc, proj_effective_scale_a, + proj_effective_scale_b); + acc += output_zp; + if (acc > int8_max) { + acc = int8_max; + } + if (acc < int8_min) { + acc = int8_min; + } + proj_output[batch * n_output + row] = acc; + } + } +} + +void PortableApplyLayerNorm(const int16_t* input, + const int16_t* layer_norm_weights, + const int32_t* bias, int32_t layer_norm_scale_a, + int32_t layer_norm_scale_b, int32_t variance_limit, + int n_batch, int n_input, int16_t* output) { + // The square of std::pow(2, 10), which is the extra factor that makes sure + // normalized values has enough resolution. + static const int kTwoToPower20 = 1 << 20; + for (int i = 0; i < n_batch; ++i) { + int64_t sum = 0; + int64_t sum_sq = 0; + for (int j = 0; j < n_input; ++j) { + const int32_t index = i * n_input + j; + int32_t val = static_cast(input[index]); + sum += val; + sum_sq += val * val; + } + int32_t mean = + static_cast(static_cast(sum) * 1024 / n_input); + // TODO(b/173994730): Avoids overflow but only works for POT n_input. + int32_t temp = kTwoToPower20 / n_input; + int64_t variance = + sum_sq * temp - static_cast(mean) * static_cast(mean); + int32_t variance2 = static_cast(variance / kTwoToPower20); + if (variance2 < 1) { + variance2 = variance_limit; + } + int32_t stddev_inverse_a; + int stddev_inverse_b; + GetInvSqrtQuantizedMultiplierExp(variance2, /*reverse_shift*/ -1, + &stddev_inverse_a, &stddev_inverse_b); + + for (int j = 0; j < n_input; ++j) { + const int32_t index = i * n_input + j; + int32_t val = static_cast(input[index]); + int32_t shifted = 1024 * val - mean; + int32_t rescaled = MultiplyByQuantizedMultiplier( + shifted, stddev_inverse_a, stddev_inverse_b); + int64_t val3 = rescaled * layer_norm_weights[j] + bias[j]; + int32_t val4 = + static_cast((val3 > 0 ? val3 + 512 : val3 - 512) / 1024); + int32_t val5 = MultiplyByQuantizedMultiplier(val4, layer_norm_scale_a, + layer_norm_scale_b + 12); + val5 = std::min(std::max(kInt16Min, val5), kInt16Max); + output[index] = static_cast(val5); + } + } +} + +void PortableApplyLayerNormFloat(const int16_t* input, + const int16_t* layer_norm_weights, + int32_t layer_norm_scale_a, + int32_t layer_norm_scale_b, + const int32_t* bias, int n_batch, int n_input, + int16_t* output) { + const int32_t int16_max = std::numeric_limits::max(); + const int32_t int16_min = std::numeric_limits::min(); + const float layer_norm_scale = + layer_norm_scale_a * + std::pow(2.0, static_cast(layer_norm_scale_b - 31)); + const float bias_scale = + static_cast(std::pow(2.0, -10)) * layer_norm_scale; + + for (int batch = 0; batch < n_batch; ++batch) { + float sum = 0.0f; + float sum_sq = 0.0f; + for (int i = 0; i < n_input; ++i) { + const int index = batch * n_input + i; + const float value = static_cast(input[index]); + sum += value; + sum_sq += value * value; + } + const float mean = sum / n_input; + float stddev_inv = 0.0f; + const float variance = sum_sq / n_input - mean * mean; + if (variance == 0) { + stddev_inv = 1.0f / std::sqrt(1e-8f); + } else { + stddev_inv = 1.0f / std::sqrt(variance); + } + for (int i = 0; i < n_input; ++i) { + const int index = batch * n_input + i; + const float normalized_value = + (static_cast(input[index]) - mean) * stddev_inv; + const float weighted_normalized_value = + normalized_value * layer_norm_weights[i] * layer_norm_scale + + bias[i] * bias_scale; + const int32_t quant_output = static_cast(round( + weighted_normalized_value * static_cast(std::pow(2, 12)))); + output[index] = std::min(int16_max, std::max(int16_min, quant_output)); + } + } +} + +void PortableMatrixScalarMultiplyAccumulate(const int8_t* matrix, + int32_t scalar, int32_t n_row, + int32_t n_col, int32_t* output) { + for (int i = 0; i < n_row; ++i) { + int32_t row_sum = 0; + for (int j = 0; j < n_col; ++j) { + row_sum += *matrix++; + } + output[i] += row_sum * scalar; + } +} + +void PortableApplySigmoid(const int16_t* input, int32_t n_batch, + int32_t n_input, int16_t* output) { + for (int batch = 0; batch < n_batch; ++batch) { + for (int c = 0; c < n_input; c++) { + using F3 = gemmlowp::FixedPoint; + using F0 = gemmlowp::FixedPoint; + const int index = batch * n_input + c; + F3 sigmoid_input = F3::FromRaw(input[index]); + F0 sigmoid_output = gemmlowp::logistic(sigmoid_input); + output[index] = sigmoid_output.raw(); + } + } +} + +void PortableApplySigmoidFloat(const int16_t* input, int32_t n_batch, + int32_t n_input, int16_t* output) { + const int32_t int16_max = std::numeric_limits::max(); + const int32_t int16_min = std::numeric_limits::min(); + for (int batch = 0; batch < n_batch; ++batch) { + for (int i = 0; i < n_input; ++i) { + const int index = batch * n_input + i; + const float float_input = + input[index] * static_cast(std::pow(2, -12)); + const float float_output = 1.0f / (1.0f + std::exp(-float_input)); + const int32_t quant_output = static_cast( + float_output * static_cast(std::pow(2, 15))); + const int32_t quant_output_clamped = + std::min(int16_max, std::max(int16_min, quant_output)); + output[index] = static_cast(quant_output_clamped); + } + } +} + +template +void PortableApplyTanhImpl(const int16_t* input, int32_t n_batch, + int32_t n_input, int16_t* output) { + using FX = gemmlowp::FixedPoint; + using F0 = gemmlowp::FixedPoint; + for (int batch = 0; batch < n_batch; ++batch) { + for (int i = 0; i < n_input; ++i) { + const int index = batch * n_input + i; + FX tanh_input = FX::FromRaw(input[index]); + F0 tanh_output = gemmlowp::tanh(tanh_input); + output[index] = tanh_output.raw(); + } + } +} + +void PortableApplyTanh(int32_t integer_bits, const int16_t* input, + int32_t n_batch, int32_t n_input, int16_t* output) { + if (integer_bits > 6) { + TFLITE_ASSERT_FALSE; + } +#define DISPATCH_TANH(i) \ + case i: \ + PortableApplyTanhImpl(input, n_batch, n_input, output); \ + break; + switch (integer_bits) { + DISPATCH_TANH(0); + DISPATCH_TANH(1); + DISPATCH_TANH(2); + DISPATCH_TANH(3); + DISPATCH_TANH(4); + DISPATCH_TANH(5); + DISPATCH_TANH(6); + default: + return; + } +#undef DISPATCH_TANH +} + +void PortableApplyTanhFloat(const int16_t* input, int32_t n_batch, + int32_t n_input, int32_t integer_bits, + int16_t* output) { + const int32_t int16_max = std::numeric_limits::max(); + const int32_t int16_min = std::numeric_limits::min(); + const double two = 2.0; + for (int batch = 0; batch < n_batch; ++batch) { + for (int i = 0; i < n_input; ++i) { + const int index = batch * n_input + i; + const float float_input = + input[index] * std::pow(two, static_cast(integer_bits)); + const float float_output = std::tanh(float_input); + const int32_t quant_output = static_cast( + float_output * static_cast(std::pow(2, 15))); + const int32_t quant_output_clamped = + std::min(int16_max, std::max(int16_min, quant_output)); + output[index] = static_cast(quant_output_clamped); + } + } +} + +void PortableCwiseMul(const int16_t* input_1, const int16_t* input_2, + int n_batch, int n_input, int shift, int16_t* output) { + for (int batch = 0; batch < n_batch; ++batch) { + for (int i = 0; i < n_input; ++i) { + const int index = batch * n_input + i; + const int16_t a = input_1[index]; + const int16_t b = input_2[index]; + const int32_t value = static_cast(a) * static_cast(b); + output[index] = + static_cast(gemmlowp::RoundingDivideByPOT(value, shift)); + } + } +} + +void PortableCwiseMul(const int16_t* input_1, const int16_t* input_2, + int32_t multiplier, int32_t shift, int32_t n_batch, + int32_t n_input, int32_t output_zp, int8_t* output) { + for (int batch = 0; batch < n_batch; ++batch) { + for (int i = 0; i < n_input; ++i) { + const int index = batch * n_input + i; + const int16_t a = input_1[index]; + const int16_t b = input_2[index]; + int32_t value = static_cast(a) * static_cast(b); + value = MultiplyByQuantizedMultiplier(value, multiplier, shift); + value -= output_zp; + value = std::min(std::max(static_cast(-128), value), + static_cast(127)); + + output[index] = static_cast(value); + } + } +} + +void PortableCwiseAdd(const int16_t* input_1, const int16_t* input_2, + int n_batch, int n_input, int16_t* output) { + for (int batch = 0; batch < n_batch; ++batch) { + for (int i = 0; i < n_input; ++i) { + const int index = batch * n_input + i; + int32_t sum = input_1[index] + input_2[index]; + const int32_t sum_clamped = std::min(kInt16Max, std::max(kInt16Min, sum)); + output[index] = static_cast(sum_clamped); + } + } +} + +float PortableVectorVectorDotProduct(const float* vector1, const float* vector2, + int v_size) { + float result = 0.0; + for (int v = 0; v < v_size; v++) { + result += *vector1++ * *vector2++; + } + return result; +} + +namespace { +inline int32_t VectorVectorDotProduct(const int16_t* vector1, + const int16_t* vector2, int v_size) { + int32_t result = 0; + for (int v = 0; v < v_size; v++) { + result += *vector1++ * *vector2++; + } + return result; +} +} // namespace + +void PortableBatchVectorBatchVectorDotProduct(const int16_t* vector1, + const int16_t* vector2, + int v_size, int n_batch, + int32_t* result) { + for (int b = 0; b < n_batch; b++) { + result[b] = VectorVectorDotProduct(vector1, vector2, v_size); + vector1 += v_size; + vector2 += v_size; + } +} + +void PortableVectorBatchVectorCwiseProductAccumulate( + const int16_t* vector, int v_size, const int16_t* batch_vector, int n_batch, + int32_t multiplier, int shift, int16_t* result) { + for (int b = 0; b < n_batch; b++) { + for (int v = 0; v < v_size; v++) { + int32_t prod = vector[v] * *batch_vector++; + prod = MultiplyByQuantizedMultiplier(prod, multiplier, shift); + int32_t output = prod + *result; + output = std::max(std::min(static_cast(32767), output), + static_cast(-32768)); + *result++ = output; + } + } +} + +void PortableSub1Vector(const float* vector, int v_size, float* result) { + for (int v = 0; v < v_size; v++) { + *result++ = 1.0f - *vector++; + } +} + +void PortableSub1Vector(const int16_t* vector, int v_size, int16_t* result) { + static const int16_t kOne = 32767; + for (int v = 0; v < v_size; v++) { + *result++ = kOne - *vector++; + } +} + +void PortableVectorScalarMultiply(const int8_t* vector, const int v_size, + const float scale, float* result) { + for (int v = 0; v < v_size; ++v) { + *result++ = scale * *vector++; + } +} + +void PortableMeanStddevNormalization(const float* __restrict__ input_vector, + float* __restrict__ output_vector, + int v_size, int n_batch) { + for (int batch = 0; batch < n_batch; ++batch) { + float sum = 0.0f; + for (int i = 0; i < v_size; ++i) { + sum += input_vector[i]; + } + const float mean = sum / v_size; + float sum_diff_sq = 0.0f; + for (int i = 0; i < v_size; ++i) { + const float diff = input_vector[i] - mean; + sum_diff_sq += diff * diff; + } + const float variance = sum_diff_sq / v_size; + constexpr float kNormalizationConstant = 1e-8f; + const float stddev_inv = + 1.0f / std::sqrt(variance + kNormalizationConstant); + for (int i = 0; i < v_size; ++i) { + output_vector[i] = (input_vector[i] - mean) * stddev_inv; + } + input_vector += v_size; + output_vector += v_size; + } +} + +void PortableTwoGateSaturatingAdd(const int8_t* input, int8_t input_zp, + const int8_t* recurrent, int8_t recurrent_zp, + int32_t input_effective_scale_a, + int32_t input_effective_scale_b, + int32_t recurrent_effective_scale_a, + int32_t recurrent_effective_scale_b, + int32_t n_batch, int32_t n_cell, + int16_t* output) { + const int32_t int16_max = std::numeric_limits::max(); + const int32_t int16_min = std::numeric_limits::min(); + for (int i = 0; i < n_batch * n_cell; ++i) { + int32_t x = static_cast(input[i]) - static_cast(input_zp); + int32_t h = + static_cast(recurrent[i]) - static_cast(recurrent_zp); + int32_t x_scaled = MultiplyByQuantizedMultiplier(x, input_effective_scale_a, + input_effective_scale_b); + int32_t h_scaled = MultiplyByQuantizedMultiplier( + h, recurrent_effective_scale_a, recurrent_effective_scale_b); + int32_t y = h_scaled + x_scaled; + if (y > int16_max) { + y = int16_max; + } + if (y < int16_min) { + y = int16_min; + } + output[i] = static_cast(y); + } +} + +} // namespace micro_tensor_utils +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_tensor_utils.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_tensor_utils.h new file mode 100644 index 00000000..673ba6a3 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/micro_tensor_utils.h @@ -0,0 +1,874 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file and the associated .cc file is branched from +// tensorflow/lite/kernels/internal/reference/portable_tensor_utils* +// TFLM needs to create its own because the original files are coupled with +// the tensor_utils module, which we cannot reuse due to its use of the +// Eigen library. + +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_MICRO_TENSOR_UTILS_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_MICRO_TENSOR_UTILS_H_ + +#include +#include +#include + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" + +#if defined(_MSC_VER) +#define __restrict__ __restrict +#endif + +namespace tflite { + +// Not all backends support CpuBackendContext usage, so forward declare to avoid +// pulling in its implementation. +// TODO(b/230666277): consider removing this since micro does not utilize it +class CpuBackendContext; + +namespace micro_tensor_utils { + +template +inline bool PortableIsZeroVector(const T* vector, int v_size) { + for (int i = 0; i < v_size; ++i) { + if (vector[i] != 0) { + return false; + } + } + return true; +} + +void PortableSymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, float* min_value, + float* max_value, float* scaling_factor); + +void PortableSymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, float min_value, + float max_value, float* scaling_factor); + +void PortableAsymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, + float* scaling_factor, int32_t* offset); + +// Multiply a matrix by a batch vector, and store results in a batch-size +// vector. +void PortableMatrixBatchVectorMultiplyAccumulate(const float* matrix, + int m_rows, int m_cols, + const float* vector, + int n_batch, float* result); + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vectors, const float* scaling_factors, + int n_batch, float* __restrict__ result); + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vectors, const float* scaling_factors, + int n_batch, float* __restrict__ result, const float* per_channel_scale, + const int32_t* input_offset, int32_t* scratch, int32_t* row_sums, + bool* compute_row_sums, CpuBackendContext* context); + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vector, const float* scaling_factors, + int n_batch, int32_t* scratch, float* __restrict__ result, + CpuBackendContext* context); + +void PortableSparseMatrixBatchVectorMultiplyAccumulate1x4( + const float* __restrict__ matrix, const int32_t* __restrict__ segments, + const int32_t* __restrict__ indices, int m_rows, int m_cols, + const float* __restrict__ vector, int n_batch, float* __restrict__ result); + +void PortableSparseMatrixBatchVectorMultiplyAccumulate( + const float* __restrict__ matrix, const uint8_t* __restrict__ ledger, + int m_rows, int m_cols, const float* __restrict__ vector, int n_batch, + float* __restrict__ result); + +void PortableSparseMatrixBatchVectorMultiplyAccumulate1x16( + const int8_t* __restrict__ matrix, const int32_t* __restrict__ segments, + const int32_t* __restrict__ indices, int m_rows, int m_cols, + const int8_t* __restrict__ vector, const int32_t* __restrict__ bias_vector, + int n_batch, const int32_t input_offset, const int32_t output_multiplier, + const int32_t output_shift, const int32_t output_offset, + const int32_t output_activation_min, const int32_t output_activation_max, + int8_t* __restrict__ result); + +void PortableSparseMatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows, + const int m_cols, const int8_t* __restrict__ vectors, + const float* scaling_factors, int n_batch, float* __restrict__ result); + +// Dot product of two vectors. +float PortableVectorVectorDotProduct(const float* vector1, const float* vector2, + int v_size); + +void PortableBatchVectorBatchVectorDotProduct(const int16_t* vector1, + const int16_t* vector2, + int v_size, int n_batch, + int32_t* result); + +void PortableVectorBatchVectorCwiseProductAccumulate( + const int16_t* vector, int v_size, const int16_t* batch_vector, int n_batch, + int32_t multiplier, int shift, int16_t* result); + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* input, const int32_t* bias, + const int8_t* input_to_gate_weights, int32_t multiplier, int32_t shift, + int32_t n_batch, int32_t n_input, int32_t n_output, int32_t output_zp, + int32_t* scratch, int16_t* output, CpuBackendContext* context); + +void PortableMatrixBatchVectorMultiplyAccumulate( + const int8_t* input, const int32_t* bias, + const int8_t* input_to_gate_weights, int32_t multiplier, int32_t shift, + int32_t n_batch, int32_t n_input, int32_t n_output, int32_t output_zp, + int32_t* scratch, int8_t* output, CpuBackendContext* context); + +void PortableMatrixBatchVectorMultiply(const int8_t* input, + int32_t input_zeropoint, + const int8_t* input_to_gate_weights, + int32_t input_to_gate_effective_scale_a, + int32_t input_to_gate_effective_scale_b, + int32_t n_batch, int32_t n_input, + int32_t n_cell, int8_t* gate_output, + int8_t gate_output_zp); + +void PortableMatrixBatchVectorMultiply( + const int16_t* hidden, const int8_t* hidden_to_output_weights, + int32_t proj_effective_scale_a, int32_t proj_effective_scale_b, + const int32_t* gate_bias, int32_t n_batch, int32_t n_hidden, + int32_t n_output, int32_t output_zp, int8_t* proj_output); + +void PortableMatrixScalarMultiplyAccumulate(const int8_t* matrix, + int32_t scalar, int32_t n_row, + int32_t n_col, int32_t* output); + +void PortableApplyLayerNorm(const int16_t* input, + const int16_t* layer_norm_weights, + const int32_t* bias, int32_t layer_norm_scale_a, + int32_t layer_norm_scale_b, int32_t variance_limit, + int n_batch, int n_input, int16_t* output); + +void PortableApplyLayerNormFloat(const int16_t* input, + const int16_t* layer_norm_weights, + int32_t layer_norm_scale_a, + int32_t layer_norm_scale_b, + const int32_t* bias, int n_batch, int n_input, + int16_t* output); + +void PortableApplySigmoid(const int16_t* input, int32_t n_batch, + int32_t n_input, int16_t* output); + +void PortableApplySigmoidFloat(const int16_t* input, int32_t n_batch, + int32_t n_input, int16_t* output); + +void PortableApplyTanh(int32_t integer_bits, const int16_t* input, + int32_t n_batch, int32_t n_input, int16_t* output); + +void PortableApplyTanhFloat(const int16_t* input, int32_t n_batch, + int32_t n_input, int32_t integer_bits, + int16_t* output); + +void PortableCwiseMul(const int16_t* input_1, const int16_t* input_2, + int n_batch, int n_input, int shift, int16_t* output); + +void PortableCwiseMul(const int16_t* input_1, const int16_t* input_2, + int32_t multiplier, int32_t shift, int32_t n_batch, + int32_t n_input, int32_t output_zp, int8_t* output); + +void PortableCwiseAdd(const int16_t* input_1, const int16_t* input_2, + int n_batch, int n_input, int16_t* output); + +template +inline void PortableCwiseClipping(T* vector, const int v_size, + const T& clipping_value) { + for (int i = 0; i < v_size; i++) { + vector[i] = std::max(std::min(clipping_value, vector[i]), + static_cast(-clipping_value)); + } +} + +// Batch vector initialization with another vector. +void PortableVectorBatchVectorAssign(const float* vector, int v_size, + int n_batch, float* batch_vector); + +// Compute "1.0f - elements of vector" (used in CIFG). +void PortableSub1Vector(const float* vector, int v_size, float* result); + +void PortableSub1Vector(const int16_t* vector, int v_size, int16_t* result); + +// Multiply all elements of vector with a scalar. +void PortableVectorScalarMultiply(const int8_t* vector, int v_size, float scale, + float* result); + +// Reduce-sum on a vector: +// input_vector: pointer to input vector. +// output_vector: pointer to vector. +// output_size: output vector size. +// reduction_size: number of consecutive elements from input vector which are +// added to get one element of output. +template +inline void PortableReductionSumVector(const INPUT* input_vector, + OUTPUT* output_vector, int output_size, + int reduction_size) { + for (int o = 0; o < output_size; o++) { + OUTPUT result = 0; + for (int r = 0; r < reduction_size; r++) { + result += input_vector[r]; + } + output_vector[o] = result; + input_vector += reduction_size; + } +} + +// Layer norm for each batch. +void PortableMeanStddevNormalization(const float* __restrict__ input_vector, + float* __restrict__ output_vector, + int v_size, int n_batch); + +// Saturate Add. +void PortableTwoGateSaturatingAdd(const int8_t* input, int8_t input_zp, + const int8_t* recurrent, int8_t recurrent_zp, + int32_t input_effective_scale_a, + int32_t input_effective_scale_b, + int32_t recurrent_effective_scale_a, + int32_t recurrent_effective_scale_b, + int32_t n_batch, int32_t n_cell, + int16_t* output); + +// Add another vector for each batch in the batch vector. +template +inline void VectorBatchVectorAdd(const T* vector, int v_size, int n_batch, + T* batch_vector) { + for (int b = 0; b < n_batch; b++) { + for (int i = 0; i < v_size; ++i) { + batch_vector[i] += vector[i]; + } + batch_vector += v_size; + } +} + +// Cwise product of two vectors. +template +inline void VectorVectorCwiseProduct(const T* vector1, const T* vector2, + int v_size, T* result) { + for (int v = 0; v < v_size; v++) { + *result++ = *vector1++ * *vector2++; + } +} + +// Cwise product of a vector and a batch-vector. +template +inline void VectorBatchVectorCwiseProduct(const T* vector, int v_size, + const T* batch_vector, int n_batch, + T* result) { + for (int b = 0; b < n_batch; b++) { + VectorVectorCwiseProduct(vector, batch_vector, v_size, result); + // Update the pointers. + result += v_size; + batch_vector += v_size; + } +} + +// Reduce-sum on a float input vector: +// input_vector: float pointer to input vector. +// output_vector: float pointer to vector. +// output_size: output vector size. +// reduction_size: number of consecutive elements from input vector which are +// added to get one element of output. +inline void ReductionSumVector(const float* input_vector, float* output_vector, + int output_size, int reduction_size) { + PortableReductionSumVector(input_vector, output_vector, output_size, + reduction_size); +} + +// Same as above but input/output is 32 bit integer. +inline void ReductionSumVector(const int32_t* input_vector, + int32_t* output_vector, int output_size, + int reduction_size) { + PortableReductionSumVector(input_vector, output_vector, output_size, + reduction_size); +} + +// Same as above but input is 8 bit integer. +inline void ReductionSumVector(const int8_t* input_vector, + int32_t* output_vector, int output_size, + int reduction_size) { + PortableReductionSumVector(input_vector, output_vector, output_size, + reduction_size); +} + +// Cwise product and accumulate of two vectors. Since it's a MAC operation, the +// assumption here is that result array is initialized to valid values. +template +inline void VectorVectorCwiseProductAccumulate(const T* __restrict__ vector1, + const T* __restrict__ vector2, + int v_size, + T* __restrict__ result) { + for (int v = 0; v < v_size; v++) { + *result++ += *vector1++ * *vector2++; + } +} + +// Batch vector initialization with another vector. +template +inline void VectorBatchVectorAssign(const T* vector, int v_size, int n_batch, + T* batch_vector) { + for (int b = 0; b < n_batch; b++) { + std::copy_n(vector, v_size, batch_vector + b * v_size); + } +} + +inline void SymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, float* min, + float* max, float* scaling_factor) { + PortableSymmetricQuantizeFloats(values, size, quantized_values, min, max, + scaling_factor); +} + +inline void SymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, float min_value, + float max_value, float* scaling_factor) { + PortableSymmetricQuantizeFloats(values, size, quantized_values, min_value, + max_value, scaling_factor); +} + +inline void AsymmetricQuantizeFloats(const float* values, const int size, + int8_t* quantized_values, + float* scaling_factor, int32_t* offset) { + PortableAsymmetricQuantizeFloats(values, size, quantized_values, + scaling_factor, offset); +} + +// Helper function to quantize floats. +// float_data_ptr input float vectors +// n_batch number of input vectors +// n_data size of a single input vector +// quantized_data_ptr (out) vector with quantized data +// scaling_factors (out) scaling factors (one per vector) +// zero_points (out) zero points (one per vector) +// do_asymmetric controls if the quantization should be asymmetric. +inline void BatchQuantizeFloats(const float* float_data_ptr, int n_batch, + int n_data, int8_t* quantized_data_ptr, + float* scaling_factors, int32_t* zero_points, + bool do_asymmetric) { + for (int b = 0; b < n_batch; ++b) { + const int offset = b * n_data; + if (do_asymmetric) { + AsymmetricQuantizeFloats(float_data_ptr + offset, n_data, + quantized_data_ptr + offset, &scaling_factors[b], + &zero_points[b]); + } else { + float unused_min, unused_max; + SymmetricQuantizeFloats(float_data_ptr + offset, n_data, + quantized_data_ptr + offset, &unused_min, + &unused_max, &scaling_factors[b]); + } + } +} + +// Check if all entries of a vector are zero for float. +inline bool IsZeroVector(const float* vector, int v_size) { + return PortableIsZeroVector(vector, v_size); +} + +// Check if all entries of a vector are zero for int8_t. +inline bool IsZeroVector(const int8_t* vector, int v_size) { + return PortableIsZeroVector(vector, v_size); +} + +// Apply Layer Normalization (https://arxiv.org/abs/1607.06450) to a Quantized +// vector. +// Parameters: +// - input: batch vector of size n_batch * n_input; 16 bit. +// - layer_norm_weights: the quantized layer normalization weights. +// - bias: the bias for the layer normalization. +// - layer_norm_scale_a: multiplier for scale factor. +// - layer_norm_scale_b: shift for scale factor. +// - variance_limit: the guard to make sure the inverse does not overflow. +// - n_batch: the number of batches. +// - n_input: the size for input and output. +// - output: the 16 bit output +inline void ApplyLayerNorm(const int16_t* input, + const int16_t* layer_norm_weights, + const int32_t* bias, int32_t layer_norm_scale_a, + int32_t layer_norm_scale_b, int32_t variance_limit, + int n_batch, int n_input, int16_t* output) { + PortableApplyLayerNorm(input, layer_norm_weights, bias, layer_norm_scale_a, + layer_norm_scale_b, variance_limit, n_batch, n_input, + output); +} + +// Same as above but the internal calculation is done in float. +inline void ApplyLayerNormFloat(const int16_t* input, + const int16_t* layer_norm_weights, + int32_t layer_norm_scale_a, + int32_t layer_norm_scale_b, const int32_t* bias, + int n_batch, int n_input, int16_t* output) { + PortableApplyLayerNormFloat(input, layer_norm_weights, layer_norm_scale_a, + layer_norm_scale_b, bias, n_batch, n_input, + output); +} + +// Apply Sigmoid to a quantized vector. +// Parameters: +// - input: batch vector of size n_batch * n_input; 16 bit. +// - n_batch: the number of batches. +// - n_input: the size for input and output. +// - output: the 16 bit output +// The input is in Q3.12 format and the output is in Q0.15 format. +inline void ApplySigmoid(const int16_t* input, int32_t n_batch, int32_t n_input, + int16_t* output) { + PortableApplySigmoid(input, n_batch, n_input, output); +} + +// Same as above but the internal calcualtion is float. +inline void ApplySigmoidFloat(const int16_t* input, int32_t n_batch, + int32_t n_input, int16_t* output) { + PortableApplySigmoidFloat(input, n_batch, n_input, output); +} + +// Apply Tanh to a quantized vector. +// Parameters: +// - integer_bits: the integer bits of the input. +// Currently supports 0, 1, 2, 3, 4, 5, 6. +// - input: batch vector of size n_batch * n_input; 16 bit. +// - n_batch: the number of batches. +// - n_input: the size for input and output. +// - output: the 16 bit output +// The input is in Qm.15-m format and the output is in Q0.15 format. +inline void ApplyTanh(int32_t integer_bits, const int16_t* input, + int32_t n_batch, int32_t n_input, int16_t* output) { + PortableApplyTanh(integer_bits, input, n_batch, n_input, output); +} + +// Apply Tanh to a quantized vector. Tbe internal calculation is in float. +// - Input has 2^(integer_bits) as scale. +// - Output has Q0.15 as scale. +inline void ApplyTanhFloat(const int16_t* input, int32_t n_batch, + int32_t n_input, int32_t integer_bits, + int16_t* output) { + PortableApplyTanhFloat(input, n_batch, n_input, integer_bits, output); +} + +// Element-wise multiplication of two quantized vectors. +// Parameters: +// - input_1: batch vector of size n_batch * n_input; 16 bit. +// - input_2: batch vector of size n_batch * n_input; 16 bit. +// - n_batch: the number of batches. +// - n_input: the size for input and output. +// - shift: the shift needed to produce the output. +// - output: the 16 bit output of size n_batch * n_input. +// Output does not need to be initialized. +inline void CwiseMul(const int16_t* input_1, const int16_t* input_2, + int n_batch, int n_input, int shift, int16_t* output) { + PortableCwiseMul(input_1, input_2, n_batch, n_input, shift, output); +} + +// Element-wise multiplication of two quantized vectors with rescaling. +// Parameters: +// - input_1: batch vector of size n_batch * n_input; 16 bit. +// - input_2: batch vector of size n_batch * n_input; 16 bit. +// - multiplier: the multiplier part of scale. +// - shift: the shift part of scale. +// - n_batch: the number of batches. +// - n_input: the size for input and output. +// - output: the 8 bit output of size n_batch * n_input. +// - output_zp: the zero point of output. +// Output does not need to be initialized. +// Multiplier ("m") and shift ("s") are connected to scale ("s") with s = m * +// 2^(s - 31). +inline void CwiseMul(const int16_t* input_1, const int16_t* input_2, + int32_t multiplier, int32_t shift, int32_t n_batch, + int32_t n_input, int32_t output_zp, int8_t* output) { + PortableCwiseMul(input_1, input_2, multiplier, shift, n_batch, n_input, + output_zp, output); +} + +// Element-wise in-place clipping of a vector. Overloaded for float, int16_t, +// int8_t. Parameters: +// - vector: vector of size v_size. +// - v_size: the size of the vector. +// - clipping_value: the value used for clipping. +inline void CwiseClipping(float* vector, const int v_size, + const float clipping_value) { + PortableCwiseClipping(vector, v_size, clipping_value); +} + +inline void CwiseClipping(int16_t* vector, const int v_size, + const int16_t clipping_value) { + PortableCwiseClipping(vector, v_size, clipping_value); +} + +inline void CwiseClipping(int8_t* vector, const int v_size, + const int8_t clipping_value) { + PortableCwiseClipping(vector, v_size, clipping_value); +} + +// Element-wise saturating addition of two quantized vectors without rescaling. +// Parameters: +// - input_1: batch vector of size n_batch * n_input; 16 bit. +// - input_2: batch vector of size n_batch * n_input; 16 bit. +// - n_batch: the number of batches. +// - n_input: the size for input and output. +// - output: the 8 bit output of size n_batch * n_input. +// Output does not need to be initialized. +inline void CwiseAdd(const int16_t* input_1, const int16_t* input_2, + int n_batch, int n_input, int16_t* output) { + PortableCwiseAdd(input_1, input_2, n_batch, n_input, output); +} + +inline void MeanStddevNormalization(const float* input_vector, + float* output_vector, int v_size, + int n_batch) { + PortableMeanStddevNormalization(input_vector, output_vector, v_size, n_batch); +} + +inline void Sub1Vector(const float* vector, int v_size, float* result) { + PortableSub1Vector(vector, v_size, result); +} + +inline void Sub1Vector(const int16_t* vector, int v_size, int16_t* result) { + PortableSub1Vector(vector, v_size, result); +} + +// Multiply all elements of vector with a scalar. +inline void VectorScalarMultiply(const int8_t* vector, int v_size, float scale, + float* result) { + PortableVectorScalarMultiply(vector, v_size, scale, result); +} + +// Saturate Add with rescale on both inputs. +inline void TwoGateSaturatingAdd(const int8_t* input, int8_t input_zp, + const int8_t* recurrent, int8_t recurrent_zp, + int32_t input_effective_scale_a, + int32_t input_effective_scale_b, + int32_t recurrent_effective_scale_a, + int32_t recurrent_effective_scale_b, + int32_t n_batch, int32_t n_cell, + int16_t* output) { + PortableTwoGateSaturatingAdd( + input, input_zp, recurrent, recurrent_zp, input_effective_scale_a, + input_effective_scale_b, recurrent_effective_scale_a, + recurrent_effective_scale_b, n_batch, n_cell, output); +} + +// Multiplies a matrix by a "batched" vector (i.e. a matrix with a batch +// dimension composed by input vectors independent from each other). The result +// of the multiplication is accumulated to the passed result buffer. +// More specifically, for a matrix M of shape [n, i] and a batched-vector +// of shape [i, batch] it will first compute the product of shape [n, batch]. +// This product will be accumulated to the result buffer. +inline void MatrixBatchVectorMultiplyAccumulate(const float* matrix, int m_rows, + int m_cols, const float* vector, + int n_batch, float* result) { + PortableMatrixBatchVectorMultiplyAccumulate(matrix, m_rows, m_cols, vector, + n_batch, result); +} + +// Same as the function above, but the matrix is a sparse tensor with block +// pattern 1x4. +// This function assumes that m_cols is a multiple of the block size (4 in this +// case) so that there's no incomplete block. +inline void MatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vector, const float* scaling_factors, + int n_batch, float* __restrict__ result) { + PortableMatrixBatchVectorMultiplyAccumulate(matrix, m_rows, m_cols, vector, + scaling_factors, n_batch, result); +} + +inline void MatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vectors, const float* scaling_factors, + int n_batch, float* __restrict__ result, const float* per_channel_scale, + const int32_t* input_offset, int32_t* scratch, int32_t* row_sums, + bool* compute_row_sums, CpuBackendContext* context) { + PortableMatrixBatchVectorMultiplyAccumulate( + matrix, m_rows, m_cols, vectors, scaling_factors, n_batch, result, + per_channel_scale, input_offset, scratch, row_sums, compute_row_sums, + context); +} + +inline void MatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vector, const float* scaling_factors, + int n_batch, int32_t* scratch, float* __restrict__ result, + CpuBackendContext* context) { + PortableMatrixBatchVectorMultiplyAccumulate(matrix, m_rows, m_cols, vector, + scaling_factors, n_batch, result); +} + +// Same as the function above, but the matrix is a sparse tensor with block +// pattern 1x4. +// This function assumes that m_cols is a multiple of the block size (4 in this +// case) so that there's no incomplete block. +inline void SparseMatrixBatchVectorMultiplyAccumulate1x4( + const float* __restrict__ matrix, const int32_t* __restrict__ segments, + const int32_t* __restrict__ indices, int m_rows, int m_cols, + const float* __restrict__ vector, int n_batch, float* __restrict__ result) { + PortableSparseMatrixBatchVectorMultiplyAccumulate1x4( + matrix, segments, indices, m_rows, m_cols, vector, n_batch, result); +} + +// Same as the function above, but the matrix is stored in block compressed +// sparse row format with block pattern 1x16 which consists of two arrays: +// 1. A matrix array stores non-zero blocks of the matrix in row major. +// 2. A ledger array stores nrows groups, one group per row. Each group starts +// with an integer representing the number of non-zero blocks for the +// corresponding row and follows with column indexes of the first element +// of each non-zero block. +// This function assumes that +// 1. m_cols is a multiple of 16 so that all blocks are full blocks. +// 2. m_cols < 254 * 16 so that block index can be represented by uint8. +inline void SparseMatrixBatchVectorMultiplyAccumulate( + const float* __restrict__ matrix, const uint8_t* __restrict__ ledger, + int m_rows, int m_cols, const float* __restrict__ vector, int n_batch, + float* __restrict__ result) { + PortableSparseMatrixBatchVectorMultiplyAccumulate( + matrix, ledger, m_rows, m_cols, vector, n_batch, result); +} + +// Same as the function above, but the matrix is a sparse tensor with block +// pattern 1x16. +// This function assumes that m_cols is a multiple of the block size (16 in this +// case) so that there's no incomplete block. Also, it assumes all offsets of +// input, output and filter are zero. +inline void SparseMatrixBatchVectorMultiplyAccumulate1x16( + const int8_t* __restrict__ matrix, const int32_t* __restrict__ segments, + const int32_t* __restrict__ indices, int m_rows, int m_cols, + const int8_t* __restrict__ vector, const int32_t* __restrict__ bias_vector, + int n_batch, const int32_t input_offset, const int32_t output_multiplier, + const int32_t output_shift, const int32_t output_offset, + const int32_t output_activation_min, const int32_t output_activation_max, + int8_t* __restrict__ result) { + PortableSparseMatrixBatchVectorMultiplyAccumulate1x16( + matrix, segments, indices, m_rows, m_cols, vector, bias_vector, n_batch, + input_offset, output_multiplier, output_shift, output_offset, + output_activation_min, output_activation_max, result); +} + +// Same as the function above, but the matrix is stored in block compressed +// sparse row format with block pattern 1x16 which consists of two arrays: +// 1. A matrix array stores non-zero blocks of the matrix in row major. +// 2. A ledger array stores nrows groups, one group per row. Each group starts +// with an integer representing the number of non-zero blocks for the +// corresponding row followed by column index of the first element of +// each non-zero block. +// This function assumes that +// 1. m_cols is a multiple of 16 so that all blocks are full blocks. +// 2. m_cols < 254 * 16 so that block index can be represented by uint8. +inline void SparseMatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows, + const int m_cols, const int8_t* __restrict__ vectors, + const float* scaling_factors, int n_batch, float* __restrict__ result) { + PortableSparseMatrixBatchVectorMultiplyAccumulate( + matrix, ledger, m_rows, m_cols, vectors, scaling_factors, n_batch, + result); +} + +// Same as the above 8, 8, 8 integer matmul except for the presence of zero +// point and non-accumulative. +// TODO(b/148688698): remove this function by folding zero point calculation in +// prepare() function. +inline void MatrixBatchVectorMultiplyAccumulate( + const int8_t* input, const int32_t* bias, + const int8_t* input_to_gate_weights, int32_t multiplier, int32_t shift, + int32_t n_batch, int32_t n_input, int32_t n_output, int32_t output_zp, + int32_t* scratch, int16_t* output, CpuBackendContext* context) { + PortableMatrixBatchVectorMultiplyAccumulate( + input, bias, input_to_gate_weights, multiplier, shift, n_batch, n_input, + n_output, output_zp, scratch, output, context); +} + +// Same as above but has 16 bit and 8 bit input and 8 bit output. +// Used in projection when hidden is 16bit. +inline void MatrixBatchVectorMultiplyAccumulate( + const int8_t* input, const int32_t* bias, + const int8_t* input_to_gate_weights, int32_t multiplier, int32_t shift, + int32_t n_batch, int32_t n_input, int32_t n_output, int32_t output_zp, + int32_t* scratch, int8_t* output, CpuBackendContext* context) { + PortableMatrixBatchVectorMultiplyAccumulate( + input, bias, input_to_gate_weights, multiplier, shift, n_batch, n_input, + n_output, output_zp, scratch, output, context); +} + +// Same as the function above, but provides separate scaling factor for the +// matrix and the vectors. The scaling factors are multiplied in the +// scaling_factor_scratch buffer. +inline void MatrixBatchVectorMultiplyAccumulate( + const int8_t* __restrict__ matrix, const int m_rows, const int m_cols, + const int8_t* __restrict__ vectors, const float matrix_scaling_factor, + const float* vector_scaling_factors, int n_batch, + float* __restrict__ result, const float* per_channel_scale, + const int32_t* input_offset, int32_t* scratch, int32_t* row_sums, + bool* compute_row_sums, float* scaling_factor_scratch, + CpuBackendContext* context) { + for (int b = 0; b < n_batch; ++b) { + scaling_factor_scratch[b] = + vector_scaling_factors[b] * matrix_scaling_factor; + } + MatrixBatchVectorMultiplyAccumulate(matrix, m_rows, m_cols, vectors, + scaling_factor_scratch, n_batch, result, + per_channel_scale, input_offset, scratch, + row_sums, compute_row_sums, context); +} + +// Multiplies a matrix with a scalar and reduce the result on each row to a +// scalar. +// Parameters: +// - matrix: matrix of size n_row * n_col +// - scalar: the scalar that is multiplied to each element in the matrix +// - n_row: the row count of the matrix +// - n_col: the column count of the matrix +// - output: the 32bit output +// Note: We do not need saturation because the int8 * int8 is safe from overflow +// in (2^31-1) / (2^14) = 131072, which is bigger than the n_row. Non-zero +// initial output value is not exceptionally large. +inline void MatrixScalarMultiplyAccumulate(const int8_t* matrix, int32_t scalar, + int32_t n_row, int32_t n_col, + int32_t* output) { + PortableMatrixScalarMultiplyAccumulate(matrix, scalar, n_row, n_col, output); +} + +// Same as the above 8, 8, 8 integer matmul except for the presence of zero +// point and non-accumulative. +// TODO(b/148688698): remove this function by folding zero point calculation in +// prepare() function. +inline void MatrixBatchVectorMultiply(const int8_t* input, + int32_t input_zeropoint, + const int8_t* input_to_gate_weights, + int32_t input_to_gate_effective_scale_a, + int32_t input_to_gate_effective_scale_b, + int32_t n_batch, int32_t n_input, + int32_t n_cell, int8_t* gate_output, + int8_t gate_output_zp) { + PortableMatrixBatchVectorMultiply( + input, input_zeropoint, input_to_gate_weights, + input_to_gate_effective_scale_a, input_to_gate_effective_scale_b, n_batch, + n_input, n_cell, gate_output, gate_output_zp); +} + +// Same as above but has 16 bit and 8 bit input and 8 bit output. +// Used in projection when hidden is 16bit. +inline void MatrixBatchVectorMultiply(const int16_t* hidden, + const int8_t* hidden_to_output_weights, + int32_t proj_effective_scale_a, + int32_t proj_effective_scale_b, + const int32_t* gate_bias, int32_t n_batch, + int32_t n_hidden, int32_t n_output, + int32_t output_zp, int8_t* proj_output) { + PortableMatrixBatchVectorMultiply(hidden, hidden_to_output_weights, + proj_effective_scale_a, + proj_effective_scale_b, gate_bias, n_batch, + n_hidden, n_output, output_zp, proj_output); +} + +// Cwise product and accumulate of a vector and a batch-vector. Since it's a MAC +// operation, the assumption here is that result array is initialized to valid +// values. +template +inline void VectorBatchVectorCwiseProductAccumulate(const T* vector, int v_size, + const T* batch_vector, + int n_batch, T* result) { + for (int b = 0; b < n_batch; b++) { + VectorVectorCwiseProductAccumulate(vector, batch_vector, v_size, result); + // Update the pointers. + result += v_size; + batch_vector += v_size; + } +} + +// Same as above, but inputs are 16bit integer and output is 16bit integer. +inline void VectorBatchVectorCwiseProductAccumulate( + const int16_t* vector, int v_size, const int16_t* batch_vector, int n_batch, + int32_t multiplier, int shift, int16_t* result) { + PortableVectorBatchVectorCwiseProductAccumulate( + vector, v_size, batch_vector, n_batch, multiplier, shift, result); +} + +// Apply Rectified Linear to elements of a vector. +inline void ApplyReluToVector(const float* vector, int v_size, float* result) { + for (int v = 0; v < v_size; v++) { + result[v] = std::max(0.0f, vector[v]); + } +} + +// Apply Rectified Linear 1 (cap to [-1;1]) to elements of a vector +inline void ApplyRelu1ToVector(const float* vector, int v_size, float* result) { + for (int v = 0; v < v_size; v++) { + result[v] = std::max(-1.0f, std::min(vector[v], 1.0f)); + } +} + +// Apply Rectified Linear 6 (cap to [0;6]) to elements of a vector +inline void ApplyRelu6ToVector(const float* vector, int v_size, float* result) { + for (int v = 0; v < v_size; v++) { + result[v] = std::max(0.0f, std::min(vector[v], 6.0f)); + } +} + +// Apply tanh to elements of a vector +inline void ApplyTanhToVector(const float* vector, int v_size, float* result) { + for (int v = 0; v < v_size; v++) { + result[v] = std::tanh(vector[v]); + } +} + +// Apply signbit to elements of a vector +inline void ApplySignbitToVector(const float* vector, int v_size, + float* result) { + for (int v = 0; v < v_size; v++) { + result[v] = std::signbit(vector[v]); + } +} + +// Apply sigmoid to elements of a vector. +inline void ApplySigmoidToVector(const float* vector, int v_size, + float* result) { + for (int v = 0; v < v_size; v++) { + result[v] = 1.0f / (1.0f + std::exp(-vector[v])); + } +} + +// Apply appropriate activation function to elements of a vector. +inline void ApplyActivationToVector(const float* vector, int v_size, + TfLiteFusedActivation activation, + float* result) { + switch (activation) { + case kTfLiteActNone: + return; + case kTfLiteActRelu: + return ApplyReluToVector(vector, v_size, result); + case kTfLiteActReluN1To1: + return ApplyRelu1ToVector(vector, v_size, result); + case kTfLiteActRelu6: + return ApplyRelu6ToVector(vector, v_size, result); + case kTfLiteActTanh: + return ApplyTanhToVector(vector, v_size, result); + case kTfLiteActSignBit: + return ApplySignbitToVector(vector, v_size, result); + case kTfLiteActSigmoid: + return ApplySigmoidToVector(vector, v_size, result); + } +} + +} // namespace micro_tensor_utils + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_KERNELS_MICRO_TENSOR_UTILS_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/mirror_pad.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/mirror_pad.cc index a19561f6..90d3bd9e 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/mirror_pad.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/mirror_pad.cc @@ -209,14 +209,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_MIRROR_PAD() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/mul.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/mul.cc index e8295197..59f006b0 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/mul.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/mul.cc @@ -61,14 +61,7 @@ TfLiteStatus MulEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration Register_MUL() { - return {/*init=*/MulInit, - /*free=*/nullptr, - /*prepare=*/MulPrepare, - /*invoke=*/MulEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(MulInit, MulPrepare, MulEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/neg.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/neg.cc index 74a95ca3..59dd8cb8 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/neg.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/neg.cc @@ -51,14 +51,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace neg TfLiteRegistration Register_NEG() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/neg::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, nullptr, neg::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/pack.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/pack.cc index 098a0482..56f3b96e 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/pack.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/pack.cc @@ -108,14 +108,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace pack TfLiteRegistration Register_PACK() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/pack::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, nullptr, pack::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/pad.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/pad.cc index 1428b16e..b645f983 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/pad.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/pad.cc @@ -223,26 +223,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace pad TfLiteRegistration Register_PAD() { - return {/*init=*/pad::Init, - /*free=*/nullptr, - /*prepare=*/pad::Prepare, - /*invoke=*/pad::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(pad::Init, pad::Prepare, pad::Eval); } // Also register Pad as PadV2. TfLiteRegistration Register_PADV2() { - return {/*init=*/pad::Init, - /*free=*/nullptr, - /*prepare=*/pad::Prepare, - /*invoke=*/pad::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(pad::Init, pad::Prepare, pad::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/pooling.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/pooling.cc index b3781636..a2ef8b62 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/pooling.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/pooling.cc @@ -88,25 +88,11 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) { } // namespace TfLiteRegistration Register_AVERAGE_POOL_2D() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/PoolingPrepare, - /*invoke=*/AverageEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, PoolingPrepare, AverageEval); } TfLiteRegistration Register_MAX_POOL_2D() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/PoolingPrepare, - /*invoke=*/MaxEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, PoolingPrepare, MaxEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/prelu.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/prelu.cc index dc0c32c0..54cc0e02 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/prelu.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/prelu.cc @@ -69,14 +69,7 @@ TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration Register_PRELU() { - return {/*init=*/PreluInit, - /*free=*/nullptr, - /*prepare=*/PreluPrepare, - /*invoke=*/PreluEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(PreluInit, PreluPrepare, PreluEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/quantize.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/quantize.cc index 97f5a004..b5eb9c3c 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/quantize.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/quantize.cc @@ -34,14 +34,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) { } // namespace TfLiteRegistration Register_QUANTIZE() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/PrepareQuantizeReference, - /*invoke=*/EvalQuantizeReference, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, PrepareQuantizeReference, + EvalQuantizeReference); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/quantize_common.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/quantize_common.cc index cca3489d..94220529 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/quantize_common.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/quantize_common.cc @@ -53,15 +53,19 @@ TfLiteStatus PrepareQuantizeReference(TfLiteContext* context, TF_LITE_ENSURE(context, affine_quantization->scale); TF_LITE_ENSURE(context, affine_quantization->scale->size == 1); - TF_LITE_ENSURE(context, - input->type == kTfLiteFloat32 || input->type == kTfLiteInt32 || - input->type == kTfLiteInt16 || input->type == kTfLiteInt8); + TF_LITE_ENSURE( + context, input->type == kTfLiteFloat32 || input->type == kTfLiteInt32 || + input->type == kTfLiteInt16 || input->type == kTfLiteInt8 || + input->type == kTfLiteUInt8); TF_LITE_ENSURE(context, output->type == kTfLiteInt8 || output->type == kTfLiteInt16 || - output->type == kTfLiteInt32); + output->type == kTfLiteInt32 || + output->type == kTfLiteUInt8); if ((input->type == kTfLiteInt16 && output->type == kTfLiteInt8) || (input->type == kTfLiteInt8 && output->type == kTfLiteInt8) || + (input->type == kTfLiteInt8 && output->type == kTfLiteUInt8) || + (input->type == kTfLiteUInt8 && output->type == kTfLiteInt8) || (input->type == kTfLiteInt8 && output->type == kTfLiteInt16) || (input->type == kTfLiteInt8 && output->type == kTfLiteInt32) || (input->type == kTfLiteInt16 && output->type == kTfLiteInt16) || @@ -109,9 +113,9 @@ TfLiteStatus EvalQuantizeReference(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorData(output)); return kTfLiteOk; default: - TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", - TfLiteTypeGetName(input->type), - TfLiteTypeGetName(output->type)); + MicroPrintf("Input %s, output %s not supported.", + TfLiteTypeGetName(input->type), + TfLiteTypeGetName(output->type)); return kTfLiteError; } } else if (input->type == kTfLiteInt32) { @@ -132,9 +136,9 @@ TfLiteStatus EvalQuantizeReference(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorData(output)); break; default: - TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", - TfLiteTypeGetName(input->type), - TfLiteTypeGetName(output->type)); + MicroPrintf("Input %s, output %s not supported.", + TfLiteTypeGetName(input->type), + TfLiteTypeGetName(output->type)); return kTfLiteError; } } else if (input->type == kTfLiteInt16) { @@ -162,9 +166,9 @@ TfLiteStatus EvalQuantizeReference(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorData(output)); return kTfLiteOk; default: - TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", - TfLiteTypeGetName(input->type), - TfLiteTypeGetName(output->type)); + MicroPrintf("Input %s, output %s not supported.", + TfLiteTypeGetName(input->type), + TfLiteTypeGetName(output->type)); return kTfLiteError; } } else if (input->type == kTfLiteInt8) { @@ -179,6 +183,13 @@ TfLiteStatus EvalQuantizeReference(TfLiteContext* context, TfLiteNode* node) { data->input_zero_point, data->quantization_params.zero_point, tflite::micro::GetTensorData(output)); break; + case kTfLiteUInt8: + reference_ops::Requantize( + tflite::micro::GetTensorData(input), size, + data->requantize_output_multiplier, data->requantize_output_shift, + data->input_zero_point, data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); + break; case kTfLiteInt16: reference_ops::Requantize( tflite::micro::GetTensorData(input), size, @@ -194,15 +205,31 @@ TfLiteStatus EvalQuantizeReference(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorData(output)); break; default: - TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", - TfLiteTypeGetName(input->type), - TfLiteTypeGetName(output->type)); + MicroPrintf("Input %s, output %s not supported.", + TfLiteTypeGetName(input->type), + TfLiteTypeGetName(output->type)); + return kTfLiteError; + } + } else if (input->type == kTfLiteUInt8) { + size_t size = ElementCount(*input->dims); + switch (output->type) { + case kTfLiteInt8: + reference_ops::Requantize( + tflite::micro::GetTensorData(input), size, + data->requantize_output_multiplier, data->requantize_output_shift, + data->input_zero_point, data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); + break; + default: + MicroPrintf("Input %s, output %s not supported.", + TfLiteTypeGetName(input->type), + TfLiteTypeGetName(output->type)); return kTfLiteError; } } else { - TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", - TfLiteTypeGetName(input->type), - TfLiteTypeGetName(output->type)); + MicroPrintf("Input %s, output %s not supported.", + TfLiteTypeGetName(input->type), + TfLiteTypeGetName(output->type)); return kTfLiteError; } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/read_variable.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/read_variable.cc index f9124f04..422c0384 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/read_variable.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/read_variable.cc @@ -81,14 +81,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace. TfLiteRegistration Register_READ_VARIABLE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce.cc index 40aed2d5..b4734f93 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce.cc @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,331 +23,50 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/types.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/reduce.h" #include "tensorflow/lite/micro/micro_utils.h" namespace tflite { -namespace ops { -namespace micro { -namespace reduce { - -constexpr int kMaxNumberOfAxis = 4; -constexpr int kMaxNumberOfReducedAxis = 2; - -struct OpData { - int32_t multiplier; - int shift; - int temp_buffer_idx; - int resolved_axis_idx; - int input_zp; - float input_scale; - int output_zp; - float output_scale; - int num_output_elements; -}; void* InitReduce(TfLiteContext* context, const char* buffer, size_t length) { - return context->AllocatePersistentBuffer(context, sizeof(OpData)); -} - -TfLiteStatus PrepareSimple(TfLiteContext* context, TfLiteNode* node) { - MicroContext* micro_context = GetMicroContext(context); - - // Inputs Tensor (dtype depends on quantization): - // [0] = Input - // [1] = Axis - TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); - - // Outputs Tensor (dtype depends on quantization): - // [0] = Output - - // Validate number of inputs and outputs - TF_LITE_ENSURE_EQ(context, node->inputs->size, 2); - TF_LITE_ENSURE_EQ(context, node->outputs->size, 1); - - // Validate axis type - TfLiteTensor* axis = micro_context->AllocateTempInputTensor(node, 1); - TF_LITE_ENSURE(context, axis != nullptr); - TF_LITE_ENSURE_TYPES_EQ(context, axis->type, kTfLiteInt32); - - if (input->type == kTfLiteInt8) { - OpData* data = static_cast(node->user_data); - TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); - const double real_multiplier = static_cast(input->params.scale) / - static_cast(output->params.scale); - QuantizeMultiplier(real_multiplier, &data->multiplier, &data->shift); - micro_context->DeallocateTempTfLiteTensor(output); - } - micro_context->DeallocateTempTfLiteTensor(axis); - micro_context->DeallocateTempTfLiteTensor(input); - return kTfLiteOk; + return context->AllocatePersistentBuffer(context, sizeof(OpDataReduce)); } TfLiteStatus PrepareMax(TfLiteContext* context, TfLiteNode* node) { - TF_LITE_ENSURE_OK(context, PrepareSimple(context, node)); - - MicroContext* micro_context = GetMicroContext(context); - OpData* op_data = static_cast(node->user_data); - TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); - TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); - TfLiteTensor* axis = micro_context->AllocateTempInputTensor(node, 1); - - op_data->input_scale = input->params.scale; - op_data->output_scale = output->params.scale; - op_data->num_output_elements = NumElements(output); - - context->RequestScratchBufferInArena(context, sizeof(int) * input->dims->size, - &op_data->temp_buffer_idx); - context->RequestScratchBufferInArena( - context, sizeof(int) * static_cast(ElementCount(*axis->dims)), - &op_data->resolved_axis_idx); - - micro_context->DeallocateTempTfLiteTensor(input); - micro_context->DeallocateTempTfLiteTensor(output); - micro_context->DeallocateTempTfLiteTensor(axis); - return kTfLiteOk; + return PrepareMaxHelper(context, node, + static_cast(node->user_data)); } TfLiteStatus PrepareMeanOrSum(TfLiteContext* context, TfLiteNode* node) { - MicroContext* micro_context = GetMicroContext(context); - TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); - OpData* op_data = reinterpret_cast(node->user_data); - TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); - if (input->type == kTfLiteInt8 || input->type == kTfLiteInt16) { - const double real_multiplier = static_cast(input->params.scale) / - static_cast(output->params.scale); - QuantizeMultiplier(real_multiplier, &op_data->multiplier, &op_data->shift); - } - - int output_size = NumElements(output); - if (input->type == kTfLiteInt8 || input->type == kTfLiteInt16) { - context->RequestScratchBufferInArena(context, output_size * sizeof(int32_t), - &op_data->temp_buffer_idx); - op_data->input_zp = input->params.zero_point; - op_data->input_scale = input->params.scale; - op_data->output_zp = output->params.zero_point; - op_data->output_scale = output->params.scale; - } - - TF_LITE_ENSURE_OK(context, PrepareSimple(context, node)); - // TODO(b/144955155): Support uint8_t(b/144955155) and int8_t(b/144955018) - micro_context->DeallocateTempTfLiteTensor(input); - micro_context->DeallocateTempTfLiteTensor(output); - return kTfLiteOk; -} - -void ResolveAxis(const int* axis_data, int axis_count, - tflite::MeanParams* op_params) { - int i = 0; - for (; i < axis_count; ++i) { - op_params->axis[i] = static_cast(axis_data[i]); - } - for (; i < 4; ++i) { - op_params->axis[i] = 1; - } - op_params->axis_count = axis_count; + return PrepareMeanOrSumHelper(context, node, + static_cast(node->user_data)); } TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) { - const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); - const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 1); - TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); - TfLiteReducerParams* params = - reinterpret_cast(node->builtin_data); - OpData* op_data = reinterpret_cast(node->user_data); - - int num_axis = static_cast(ElementCount(*axis->dims)); - int temp_index[kMaxNumberOfAxis]; - int resolved_axis[kMaxNumberOfReducedAxis]; - - tflite::MeanParams op_params; - ResolveAxis(tflite::micro::GetTensorData(axis), num_axis, &op_params); - - // Special case mean implementation exists for 4D mean across axes 1 and 2. - bool special_case_4d_axes_1_and_2 = - input->dims->size == 4 && op_params.axis_count == 2 && - ((op_params.axis[0] == 1 && op_params.axis[1] == 2) || - (op_params.axis[0] == 2 && op_params.axis[1] == 1)); - - switch (input->type) { - case kTfLiteFloat32: { - // Defer to specialized implementation for 4D Mean across axes 1 & 2. - if (params->keep_dims && special_case_4d_axes_1_and_2) { - reference_ops::Mean(op_params, tflite::micro::GetTensorShape(input), - tflite::micro::GetTensorData(input), - tflite::micro::GetTensorShape(output), - tflite::micro::GetTensorData(output)); - } else { - TF_LITE_ENSURE( - context, - reference_ops::Mean( - tflite::micro::GetTensorData(input), input->dims->data, - input->dims->size, tflite::micro::GetTensorData(output), - output->dims->data, output->dims->size, - tflite::micro::GetTensorData(axis), num_axis, - params->keep_dims, temp_index, resolved_axis, - tflite::micro::GetTensorData(output))); - } - } break; - case kTfLiteInt8: { - // Defer to specialized implementation for 4D Mean across axes 1 & 2. - if (params->keep_dims && special_case_4d_axes_1_and_2) { - reference_integer_ops::Mean( - op_params, op_data->multiplier, op_data->shift, - tflite::micro::GetTensorShape(input), - tflite::micro::GetTensorData(input), op_data->input_zp, - tflite::micro::GetTensorShape(output), - tflite::micro::GetTensorData(output), op_data->output_zp); - } else if (op_data->input_zp == op_data->output_zp && - op_data->input_scale == op_data->output_scale) { - int32_t* temp_buffer = static_cast( - context->GetScratchBuffer(context, op_data->temp_buffer_idx)); - TF_LITE_ENSURE( - context, - reference_ops::Mean( - tflite::micro::GetTensorData(input), input->dims->data, - input->dims->size, tflite::micro::GetTensorData(output), - output->dims->data, output->dims->size, - tflite::micro::GetTensorData(axis), num_axis, - params->keep_dims, temp_index, resolved_axis, temp_buffer)); - } else { - int32_t* temp_buffer = static_cast( - context->GetScratchBuffer(context, op_data->temp_buffer_idx)); - TF_LITE_ENSURE( - context, - reference_ops::QuantizedMeanOrSum( - tflite::micro::GetTensorData(input), op_data->input_zp, - op_data->input_scale, input->dims->data, input->dims->size, - tflite::micro::GetTensorData(output), - op_data->output_zp, op_data->output_scale, output->dims->data, - output->dims->size, tflite::micro::GetTensorData(axis), - num_axis, params->keep_dims, temp_index, resolved_axis, - temp_buffer, false)); - } - } break; - case kTfLiteInt16: { - // Defer to specialized implementation for 4D Mean across axes 1 & 2. - if (params->keep_dims && special_case_4d_axes_1_and_2) { - reference_integer_ops::Mean( - op_params, op_data->multiplier, op_data->shift, - tflite::micro::GetTensorShape(input), - tflite::micro::GetTensorData(input), op_data->input_zp, - tflite::micro::GetTensorShape(output), - tflite::micro::GetTensorData(output), op_data->output_zp); - } else if (op_data->input_zp == op_data->output_zp && - op_data->input_scale == op_data->output_scale) { - int32_t* temp_buffer = static_cast( - context->GetScratchBuffer(context, op_data->temp_buffer_idx)); - TF_LITE_ENSURE( - context, - reference_ops::Mean(tflite::micro::GetTensorData(input), - input->dims->data, input->dims->size, - tflite::micro::GetTensorData(output), - output->dims->data, output->dims->size, - tflite::micro::GetTensorData(axis), - num_axis, params->keep_dims, temp_index, - resolved_axis, temp_buffer)); - } else { - int32_t* temp_buffer = static_cast( - context->GetScratchBuffer(context, op_data->temp_buffer_idx)); - TF_LITE_ENSURE( - context, - reference_ops::QuantizedMeanOrSum( - tflite::micro::GetTensorData(input), op_data->input_zp, - op_data->input_scale, input->dims->data, input->dims->size, - tflite::micro::GetTensorData(output), - op_data->output_zp, op_data->output_scale, output->dims->data, - output->dims->size, tflite::micro::GetTensorData(axis), - num_axis, params->keep_dims, temp_index, resolved_axis, - temp_buffer, false)); - } - } break; - default: - TF_LITE_ENSURE_MSG(context, false, - "Currently, only float32, int8 or uint8 input type " - "is supported."); - } - return kTfLiteOk; + return EvalMeanHelper(context, node, + static_cast(node->user_data)); } TfLiteStatus EvalMax(TfLiteContext* context, TfLiteNode* node) { - const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); - const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 1); - TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); - TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); - TfLiteReducerParams* params = - static_cast(node->builtin_data); - OpData* op_data = static_cast(node->user_data); - - // Interpret an axis tensor with null dimensions as a scalar - int num_axis = static_cast(ElementCount(*axis->dims)); - int* temp_buffer = static_cast( - context->GetScratchBuffer(context, op_data->temp_buffer_idx)); - int* resolved_axis = static_cast( - context->GetScratchBuffer(context, op_data->resolved_axis_idx)); - switch (input->type) { - case kTfLiteFloat32: - TF_LITE_ENSURE( - context, - reference_ops::ReduceGeneric( - tflite::micro::GetTensorData(input), input->dims->data, - input->dims->size, tflite::micro::GetTensorData(output), - output->dims->data, output->dims->size, - tflite::micro::GetTensorData(axis), num_axis, - params->keep_dims, temp_buffer, resolved_axis, - std::numeric_limits::lowest(), - [](const float current, const float in) -> float { - return (in > current) ? in : current; - })); - break; - case kTfLiteInt8: - TF_LITE_ENSURE_EQ(context, static_cast(op_data->input_scale), - static_cast(op_data->output_scale)); - TF_LITE_ENSURE_EQ(context, op_data->input_zp, op_data->output_zp); - TF_LITE_ENSURE( - context, - reference_ops::ReduceGeneric( - tflite::micro::GetTensorData(input), input->dims->data, - input->dims->size, tflite::micro::GetTensorData(output), - output->dims->data, output->dims->size, - tflite::micro::GetTensorData(axis), num_axis, - params->keep_dims, temp_buffer, resolved_axis, - std::numeric_limits::lowest(), - [](const int8_t current, const int8_t in) -> int8_t { - return (in > current) ? in : current; - })); - break; - default: - TF_LITE_KERNEL_LOG(context, - "Only float32 and int8 types are supported.\n"); - return kTfLiteError; - } - return kTfLiteOk; + OpDataReduce* op_data = static_cast(node->user_data); + return EvalMaxHelper(context, node, op_data); } -} // namespace reduce +TfLiteStatus EvalSum(TfLiteContext* context, TfLiteNode* node) { + return EvalSumHelper(context, node, + static_cast(node->user_data)); +} TfLiteRegistration Register_MEAN() { - return {/*init=*/reduce::InitReduce, - /*free=*/nullptr, - /*prepare=*/reduce::PrepareMeanOrSum, - /*invoke=*/reduce::EvalMean, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(InitReduce, PrepareMeanOrSum, EvalMean); } TfLiteRegistration Register_REDUCE_MAX() { - return {/*init=*/reduce::InitReduce, - /*free=*/nullptr, - /*prepare=*/reduce::PrepareMax, - /*invoke=*/reduce::EvalMax, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(InitReduce, PrepareMax, EvalMax); +} + +TfLiteRegistration Register_SUM() { + return tflite::micro::RegisterOp(InitReduce, PrepareMeanOrSum, EvalSum); } -} // namespace micro -} // namespace ops } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce.h new file mode 100644 index 00000000..8d524069 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce.h @@ -0,0 +1,64 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_REDUCE_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_REDUCE_H_ + +#include + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/types.h" + +namespace tflite { + +extern const int kMaxNumberOfAxis; +extern const int kMaxNumberOfReducedAxis; + +struct OpDataReduce { + int32_t multiplier; + int shift; + int temp_buffer_idx; + int resolved_axis_idx; + int input_zp; + float input_scale; + int output_zp; + float output_scale; + int num_output_elements; +}; + +TfLiteStatus PrepareMaxHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data); + +TfLiteStatus PrepareMeanOrSumHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data); + +TfLiteStatus EvalMaxHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data); +TfLiteStatus EvalMeanHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data); +TfLiteStatus EvalSumHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data); + +void ReduceResolveAxis(const int* axis_data, int axis_count, + MeanParams* op_params); + +TfLiteRegistration Register_MEAN(); +TfLiteRegistration Register_REDUCE_MAX(); +TfLiteRegistration Register_SUM(); + +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_KERNELS_REDUCE_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce_common.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce_common.cc new file mode 100644 index 00000000..a6f940c6 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/reduce_common.cc @@ -0,0 +1,374 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/integer_ops/mean.h" +#include "tensorflow/lite/kernels/internal/reference/reduce.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/internal/types.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/reduce.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" +#include "tensorflow/lite/micro/micro_utils.h" + +namespace tflite { + +const int kMaxNumberOfAxis = 4; +const int kMaxNumberOfReducedAxis = 2; + +TfLiteStatus PrepareSimple(TfLiteContext* context, TfLiteNode* node, + int32_t* multiplier, int* shift) { + MicroContext* micro_context = GetMicroContext(context); + + // Inputs Tensor (dtype depends on quantization): + // [0] = Input + // [1] = Axis + TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); + + // Outputs Tensor (dtype depends on quantization): + // [0] = Output + + // Validate number of inputs and outputs + TF_LITE_ENSURE_EQ(context, node->inputs->size, 2); + TF_LITE_ENSURE_EQ(context, node->outputs->size, 1); + + // Validate axis type + TfLiteTensor* axis = micro_context->AllocateTempInputTensor(node, 1); + TF_LITE_ENSURE(context, axis != nullptr); + TF_LITE_ENSURE_TYPES_EQ(context, axis->type, kTfLiteInt32); + + if (input->type == kTfLiteInt8) { + TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); + const double real_multiplier = static_cast(input->params.scale) / + static_cast(output->params.scale); + QuantizeMultiplier(real_multiplier, multiplier, shift); + micro_context->DeallocateTempTfLiteTensor(output); + } + micro_context->DeallocateTempTfLiteTensor(axis); + micro_context->DeallocateTempTfLiteTensor(input); + return kTfLiteOk; +} + +TfLiteStatus PrepareMaxHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data) { + TF_LITE_ENSURE_OK(context, PrepareSimple(context, node, &op_data->multiplier, + &op_data->shift)); + + MicroContext* micro_context = GetMicroContext(context); + TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); + TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); + TfLiteTensor* axis = micro_context->AllocateTempInputTensor(node, 1); + + op_data->input_scale = input->params.scale; + op_data->output_scale = output->params.scale; + op_data->num_output_elements = NumElements(output); + + context->RequestScratchBufferInArena(context, sizeof(int) * input->dims->size, + &op_data->temp_buffer_idx); + context->RequestScratchBufferInArena( + context, sizeof(int) * static_cast(ElementCount(*axis->dims)), + &op_data->resolved_axis_idx); + + micro_context->DeallocateTempTfLiteTensor(input); + micro_context->DeallocateTempTfLiteTensor(output); + micro_context->DeallocateTempTfLiteTensor(axis); + return kTfLiteOk; +} + +TfLiteStatus PrepareMeanOrSumHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data) { + MicroContext* micro_context = GetMicroContext(context); + TfLiteTensor* input = micro_context->AllocateTempInputTensor(node, 0); + TfLiteTensor* output = micro_context->AllocateTempOutputTensor(node, 0); + if (input->type == kTfLiteInt8 || input->type == kTfLiteInt16) { + const double real_multiplier = static_cast(input->params.scale) / + static_cast(output->params.scale); + QuantizeMultiplier(real_multiplier, &op_data->multiplier, &op_data->shift); + } + + int output_size = NumElements(output); + if (input->type == kTfLiteInt8 || input->type == kTfLiteInt16) { + context->RequestScratchBufferInArena(context, output_size * sizeof(int32_t), + &op_data->temp_buffer_idx); + op_data->input_zp = input->params.zero_point; + op_data->input_scale = input->params.scale; + op_data->output_zp = output->params.zero_point; + op_data->output_scale = output->params.scale; + } + + TF_LITE_ENSURE_OK( + context, + PrepareSimple(context, node, &(op_data->multiplier), &(op_data->shift))); + // TODO(b/144955155): Support uint8_t(b/144955155) and int8_t(b/144955018) + micro_context->DeallocateTempTfLiteTensor(input); + micro_context->DeallocateTempTfLiteTensor(output); + return kTfLiteOk; +} + +void ResolveAxis(const int* axis_data, int axis_count, + tflite::MeanParams* op_params) { + int i = 0; + for (; i < axis_count; ++i) { + op_params->axis[i] = static_cast(axis_data[i]); + } + for (; i < 4; ++i) { + op_params->axis[i] = 1; + } + op_params->axis_count = axis_count; +} + +TfLiteStatus EvalMeanHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data) { + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 1); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); + TfLiteReducerParams* params = + reinterpret_cast(node->builtin_data); + + int num_axis = static_cast(ElementCount(*axis->dims)); + int temp_index[kMaxNumberOfAxis]; + int resolved_axis[kMaxNumberOfReducedAxis]; + + tflite::MeanParams op_params; + ResolveAxis(tflite::micro::GetTensorData(axis), num_axis, &op_params); + + // Special case mean implementation exists for 4D mean across axes 1 and 2. + bool special_case_4d_axes_1_and_2 = + input->dims->size == 4 && op_params.axis_count == 2 && + ((op_params.axis[0] == 1 && op_params.axis[1] == 2) || + (op_params.axis[0] == 2 && op_params.axis[1] == 1)); + + switch (input->type) { + case kTfLiteFloat32: { + // Defer to specialized implementation for 4D Mean across axes 1 & 2. + if (params->keep_dims && special_case_4d_axes_1_and_2) { + reference_ops::Mean(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); + } else { + TF_LITE_ENSURE( + context, + reference_ops::Mean( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_index, resolved_axis, + tflite::micro::GetTensorData(output))); + } + } break; + case kTfLiteInt8: { + // Defer to specialized implementation for 4D Mean across axes 1 & 2. + if (params->keep_dims && special_case_4d_axes_1_and_2) { + reference_integer_ops::Mean( + op_params, op_data->multiplier, op_data->shift, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), op_data->input_zp, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), op_data->output_zp); + } else if (op_data->input_zp == op_data->output_zp && + op_data->input_scale == op_data->output_scale) { + int32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::Mean( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_index, resolved_axis, temp_buffer)); + } else { + int32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::QuantizedMeanOrSum( + tflite::micro::GetTensorData(input), op_data->input_zp, + op_data->input_scale, input->dims->data, input->dims->size, + tflite::micro::GetTensorData(output), + op_data->output_zp, op_data->output_scale, output->dims->data, + output->dims->size, tflite::micro::GetTensorData(axis), + num_axis, params->keep_dims, temp_index, resolved_axis, + temp_buffer, false)); + } + } break; + case kTfLiteInt16: { + // Defer to specialized implementation for 4D Mean across axes 1 & 2. + if (params->keep_dims && special_case_4d_axes_1_and_2) { + reference_integer_ops::Mean( + op_params, op_data->multiplier, op_data->shift, + tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), op_data->input_zp, + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), op_data->output_zp); + } else if (op_data->input_zp == op_data->output_zp && + op_data->input_scale == op_data->output_scale) { + int32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::Mean(tflite::micro::GetTensorData(input), + input->dims->data, input->dims->size, + tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), + num_axis, params->keep_dims, temp_index, + resolved_axis, temp_buffer)); + } else { + int32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::QuantizedMeanOrSum( + tflite::micro::GetTensorData(input), op_data->input_zp, + op_data->input_scale, input->dims->data, input->dims->size, + tflite::micro::GetTensorData(output), + op_data->output_zp, op_data->output_scale, output->dims->data, + output->dims->size, tflite::micro::GetTensorData(axis), + num_axis, params->keep_dims, temp_index, resolved_axis, + temp_buffer, false)); + } + } break; + default: + TF_LITE_ENSURE_MSG(context, false, + "Currently, only float32, int8 or int16 input type " + "is supported."); + } + return kTfLiteOk; +} + +TfLiteStatus EvalMaxHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data) { + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 1); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); + TfLiteReducerParams* params = + static_cast(node->builtin_data); + + // Interpret an axis tensor with null dimensions as a scalar + int num_axis = static_cast(ElementCount(*axis->dims)); + int* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + int* resolved_axis = static_cast( + context->GetScratchBuffer(context, op_data->resolved_axis_idx)); + switch (input->type) { + case kTfLiteFloat32: + TF_LITE_ENSURE( + context, + reference_ops::ReduceGeneric( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_buffer, resolved_axis, + std::numeric_limits::lowest(), + [](const float current, const float in) -> float { + return (in > current) ? in : current; + })); + break; + case kTfLiteInt8: + TF_LITE_ENSURE_EQ(context, static_cast(op_data->input_scale), + static_cast(op_data->output_scale)); + TF_LITE_ENSURE_EQ(context, op_data->input_zp, op_data->output_zp); + TF_LITE_ENSURE( + context, + reference_ops::ReduceGeneric( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_buffer, resolved_axis, + std::numeric_limits::lowest(), + [](const int8_t current, const int8_t in) -> int8_t { + return (in > current) ? in : current; + })); + break; + default: + MicroPrintf("Only float32 and int8 types are supported."); + return kTfLiteError; + } + return kTfLiteOk; +} + +TfLiteStatus EvalSumHelper(TfLiteContext* context, TfLiteNode* node, + OpDataReduce* op_data) { + const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); + const TfLiteEvalTensor* axis = tflite::micro::GetEvalInput(context, node, 1); + TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); + TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); + TfLiteReducerParams* params = + static_cast(node->builtin_data); + + // Interpret an axis tensor with null dimensions as a scalar. + int num_axis = static_cast(ElementCount(*axis->dims)); + int temp_index[kMaxNumberOfAxis]; + int resolved_axis[kMaxNumberOfReducedAxis]; + + switch (input->type) { + case kTfLiteFloat32: { + TF_LITE_ENSURE( + context, + reference_ops::ReduceGeneric( + tflite::micro::GetTensorData(input), input->dims->data, + input->dims->size, tflite::micro::GetTensorData(output), + output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_index, resolved_axis, /*init_value=*/0.f, + [](const float current, const float in) -> float { + return in + current; + })); + } break; + case kTfLiteInt8: { + int32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::QuantizedMeanOrSum( + tflite::micro::GetTensorData(input), op_data->input_zp, + op_data->input_scale, input->dims->data, input->dims->size, + tflite::micro::GetTensorData(output), op_data->output_zp, + op_data->output_scale, output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_index, resolved_axis, temp_buffer, + /*compute_sum=*/true)); + } break; + case kTfLiteInt16: { + int32_t* temp_buffer = static_cast( + context->GetScratchBuffer(context, op_data->temp_buffer_idx)); + TF_LITE_ENSURE( + context, + reference_ops::QuantizedMeanOrSum( + tflite::micro::GetTensorData(input), op_data->input_zp, + op_data->input_scale, input->dims->data, input->dims->size, + tflite::micro::GetTensorData(output), op_data->output_zp, + op_data->output_scale, output->dims->data, output->dims->size, + tflite::micro::GetTensorData(axis), num_axis, + params->keep_dims, temp_index, resolved_axis, temp_buffer, + /*compute_sum=*/true)); + } break; + default: + MicroPrintf("Only float32, int8, and int16 types are supported."); + return kTfLiteError; + } + return kTfLiteOk; +} + +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/reshape.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/reshape.cc index d14ed82e..832ba261 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/reshape.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/reshape.cc @@ -110,14 +110,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace reshape TfLiteRegistration Register_RESHAPE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/reshape::Prepare, - /*invoke=*/reshape::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, reshape::Prepare, reshape::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/resize_bilinear.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/resize_bilinear.cc index 55c23846..a90057b9 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/resize_bilinear.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/resize_bilinear.cc @@ -111,14 +111,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_RESIZE_BILINEAR() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc index a02159af..ce507445 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc @@ -117,14 +117,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace resize_nearest_neighbor TfLiteRegistration Register_RESIZE_NEAREST_NEIGHBOR() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/resize_nearest_neighbor::Prepare, - /*invoke=*/resize_nearest_neighbor::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, resize_nearest_neighbor::Prepare, + resize_nearest_neighbor::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/round.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/round.cc index 76d8e6bf..0bda8783 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/round.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/round.cc @@ -68,14 +68,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace round TfLiteRegistration Register_ROUND() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/round::Prepare, - /*invoke=*/round::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, round::Prepare, round::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/shape.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/shape.cc index df962f62..02f663a8 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/shape.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/shape.cc @@ -60,14 +60,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_SHAPE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/slice.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/slice.cc index 40d9fdd7..212cf47f 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/slice.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/slice.cc @@ -151,14 +151,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_SLICE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax.cc index f6a30010..c2cee3c5 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax.cc @@ -83,14 +83,7 @@ TfLiteStatus SoftmaxEval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_SOFTMAX() { - return {/*init=*/SoftmaxInit, - /*free=*/nullptr, - /*prepare=*/SoftmaxPrepare, - /*invoke=*/SoftmaxEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(SoftmaxInit, SoftmaxPrepare, SoftmaxEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax.h index 8d605eab..7096d202 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax.h @@ -1,4 +1,4 @@ -/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,6 +23,13 @@ namespace tflite { void* SoftmaxInit(TfLiteContext* context, const char* buffer, size_t length); +// Common helper function to SoftmaxPrepare. +TfLiteStatus CalculateSoftmaxParams(TfLiteContext* context, + const TfLiteTensor* input, + TfLiteTensor* output, + const TfLiteSoftmaxParams* params, + SoftmaxParams* op_data); + TfLiteStatus SoftmaxPrepare(TfLiteContext* context, TfLiteNode* node); // This is the most generic TfLiteRegistration. The actual supported types may @@ -30,7 +37,7 @@ TfLiteStatus SoftmaxPrepare(TfLiteContext* context, TfLiteNode* node); // (reference or optimized) must define this function. TfLiteRegistration Register_SOFTMAX(); -#if defined(XTENSA) +#if defined(XTENSA) || defined(CMSIS_NN) // Returns a TfLiteRegistration struct for kernel variant that only supports // int8 input and int16 output. TfLiteRegistration Register_SOFTMAX_INT8_INT16(); @@ -40,6 +47,23 @@ inline TfLiteRegistration Register_SOFTMAX_INT8_INT16() { } #endif +#if defined(CMSIS_NN) +// Returns a TfLiteRegistration struct for kernel variant that only supports +// int8 input/output and uses the latency optimized implementations. +TfLiteRegistration Register_SOFTMAX_INT8(); + +// Returns a TfLiteRegistration struct for kernel variant that only supports +// int16 input/output and uses the latency optimized implementations. +TfLiteRegistration Register_SOFTMAX_INT16(); + +#else +inline TfLiteRegistration Register_SOFTMAX_INT8() { return Register_SOFTMAX(); } + +inline TfLiteRegistration Register_SOFTMAX_INT16() { + return Register_SOFTMAX(); +} +#endif + } // namespace tflite #endif // TENSORFLOW_LITE_MICRO_KERNELS_SOFTMAX_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax_common.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax_common.cc index d93f5f26..b5378dae 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax_common.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/softmax_common.cc @@ -1,4 +1,4 @@ -/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -28,11 +28,59 @@ namespace { // Softmax parameter data that persists in user_data const int kInt16LUTArraySize = 513; +TfLiteStatus InitializeLutForInt16(TfLiteContext* context, + const TfLiteTensor* input, + TfLiteTensor* output, + SoftmaxParams* op_data) { + // Only allocate LUTs for KTfLiteInt16 data type + if (input->type == kTfLiteInt16) { + void* raw_exp_lut = context->AllocatePersistentBuffer( + context, sizeof(int16_t) * kInt16LUTArraySize); + TF_LITE_ENSURE(context, raw_exp_lut != nullptr); + op_data->exp_lut = reinterpret_cast(raw_exp_lut); + void* one_over_one_plus_x_lut = context->AllocatePersistentBuffer( + context, sizeof(int16_t) * kInt16LUTArraySize); + TF_LITE_ENSURE(context, one_over_one_plus_x_lut != nullptr); + op_data->one_over_one_plus_x_lut = + reinterpret_cast(one_over_one_plus_x_lut); + } + + if (output->type == kTfLiteInt16) { + TF_LITE_ENSURE(context, + input->type == kTfLiteInt8 || input->type == kTfLiteInt16); + } else { + TF_LITE_ENSURE_EQ(context, input->type, output->type); + } + + // Populate LUT if required + if (input->type == kTfLiteInt16) { + TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); + // exp LUT only used on negative values + // we consider exp(-10.0) is insignificant to accumulation + gen_lut( + [](float value) { return std::exp(value); }, -10.0f, 0.0f, -1.0f, 1.0f, + op_data->exp_lut); + gen_lut( + [](float value) { return 1.0f / (1.0f + value); }, 0.0f, 1.0f, -1.0f, + 1.0f, op_data->one_over_one_plus_x_lut); + op_data->zero_point = output->params.zero_point; + op_data->scale = output->params.scale; + } + + return kTfLiteOk; +} + +} // namespace + TfLiteStatus CalculateSoftmaxParams(TfLiteContext* context, const TfLiteTensor* input, TfLiteTensor* output, const TfLiteSoftmaxParams* params, SoftmaxParams* op_data) { + if (InitializeLutForInt16(context, input, output, op_data) != kTfLiteOk) { + return kTfLiteError; + } + if (input->type == kTfLiteInt8 || input->type == kTfLiteInt16) { if (input->type == kTfLiteInt16) { TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); @@ -83,8 +131,6 @@ TfLiteStatus CalculateSoftmaxParams(TfLiteContext* context, return kTfLiteOk; } -} // namespace - void* SoftmaxInit(TfLiteContext* context, const char* buffer, size_t length) { TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); return context->AllocatePersistentBuffer(context, sizeof(SoftmaxParams)); @@ -103,40 +149,6 @@ TfLiteStatus SoftmaxPrepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE(context, node->user_data != nullptr); SoftmaxParams* op_data = static_cast(node->user_data); - // Only allocate LUTs for KTfLiteInt16 data type - if (input->type == kTfLiteInt16) { - void* raw_exp_lut = context->AllocatePersistentBuffer( - context, sizeof(int16_t) * kInt16LUTArraySize); - TF_LITE_ENSURE(context, raw_exp_lut != nullptr); - op_data->exp_lut = reinterpret_cast(raw_exp_lut); - void* one_over_one_plus_x_lut = context->AllocatePersistentBuffer( - context, sizeof(int16_t) * kInt16LUTArraySize); - TF_LITE_ENSURE(context, one_over_one_plus_x_lut != nullptr); - op_data->one_over_one_plus_x_lut = - reinterpret_cast(one_over_one_plus_x_lut); - } - - if (output->type == kTfLiteInt16) { - TF_LITE_ENSURE(context, - input->type == kTfLiteInt8 || input->type == kTfLiteInt16); - } else { - TF_LITE_ENSURE_EQ(context, input->type, output->type); - } - - // Populate LUT if required - if (input->type == kTfLiteInt16) { - TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); - // exp LUT only used on negative values - // we consider exp(-10.0) is insignificant to accumulation - gen_lut( - [](float value) { return std::exp(value); }, -10.0f, 0.0f, -1.0f, 1.0f, - op_data->exp_lut); - gen_lut( - [](float value) { return 1.0f / (1.0f + value); }, 0.0f, 1.0f, -1.0f, - 1.0f, op_data->one_over_one_plus_x_lut); - op_data->zero_point = output->params.zero_point; - op_data->scale = output->params.scale; - } auto* params = static_cast(node->builtin_data); auto ret_val = diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/space_to_batch_nd.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/space_to_batch_nd.cc index 4e01becb..21f81312 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/space_to_batch_nd.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/space_to_batch_nd.cc @@ -114,14 +114,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace. TfLiteRegistration Register_SPACE_TO_BATCH_ND() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/space_to_depth.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/space_to_depth.cc index 9c0cc445..30519b27 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/space_to_depth.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/space_to_depth.cc @@ -121,14 +121,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_SPACE_TO_DEPTH() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/split.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/split.cc index 5d90d983..06584d45 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/split.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/split.cc @@ -120,14 +120,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace split TfLiteRegistration Register_SPLIT() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/split::Prepare, - /*invoke=*/split::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, split::Prepare, split::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/split_v.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/split_v.cc index c1c41124..3ea35130 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/split_v.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/split_v.cc @@ -121,14 +121,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace split_v TfLiteRegistration Register_SPLIT_V() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/split_v::Prepare, - /*invoke=*/split_v::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, split_v::Prepare, split_v::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/squared_difference.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/squared_difference.cc new file mode 100644 index 00000000..ca924e26 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/squared_difference.cc @@ -0,0 +1,247 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/binary_function.h" +#include "tensorflow/lite/kernels/internal/reference/integer_ops/add.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/micro_context.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" + +namespace tflite { +namespace { +constexpr int kInputTensor1 = 0; +constexpr int kInputTensor2 = 1; +constexpr int kOutputTensor = 0; + +struct OpData { + bool requires_broadcast; + ArithmeticParams arithmetic_params; +}; + +template +T SquaredDifference(T input1, T input2) { + const T difference = input1 - input2; + return difference * difference; +} + +void* SquaredDifferenceInit(TfLiteContext* context, const char* buffer, + size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer(context, sizeof(OpData)); +} + +TfLiteStatus SquaredDifferencePrepare(TfLiteContext* context, + TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + OpData* data = reinterpret_cast(node->user_data); + data->requires_broadcast = false; + + TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + MicroContext* micro_context = GetMicroContext(context); + + TfLiteTensor* input1 = + micro_context->AllocateTempInputTensor(node, kInputTensor1); + TF_LITE_ENSURE(context, input1 != nullptr); + TfLiteTensor* input2 = + micro_context->AllocateTempInputTensor(node, kInputTensor2); + TF_LITE_ENSURE(context, input2 != nullptr); + TfLiteTensor* output = + micro_context->AllocateTempOutputTensor(node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + TF_LITE_ENSURE_TYPES_EQ(context, input1->type, input2->type); + output->type = input2->type; + + // Ensure the quantization parameters are equivalent. + if (input1->type == kTfLiteInt8) { + const auto& input1_quantization_params = input1->params; + const auto& input2_quantization_params = input2->params; + const auto& output_quantization_params = output->params; + const int32_t integer_type_min = std::numeric_limits::min(); + const int32_t integer_type_max = std::numeric_limits::max(); + TF_LITE_ENSURE(context, + input1_quantization_params.zero_point >= integer_type_min); + TF_LITE_ENSURE(context, + input1_quantization_params.zero_point <= integer_type_max); + TF_LITE_ENSURE(context, + input2_quantization_params.zero_point >= integer_type_min); + TF_LITE_ENSURE(context, + input2_quantization_params.zero_point <= integer_type_max); + TF_LITE_ENSURE(context, + output_quantization_params.zero_point >= integer_type_min); + TF_LITE_ENSURE(context, + output_quantization_params.zero_point <= integer_type_max); + data->arithmetic_params.input1_offset = + -input1_quantization_params.zero_point; + data->arithmetic_params.input2_offset = + -input2_quantization_params.zero_point; + data->arithmetic_params.output_offset = + output_quantization_params.zero_point; + + // shift to make integer for scales. + // 7 is selected so that maximum shifted result 255^2 * (1 << (7 * 2 )) + // does not overflow signed 32-bit integer + data->arithmetic_params.left_shift = 7; + const double twice_max_input_scale = + 2.0 * static_cast(std::max(input1_quantization_params.scale, + input2_quantization_params.scale)); + const double real_input1_multiplier = + static_cast(input1_quantization_params.scale) / + twice_max_input_scale; + double real_input2_multiplier = + static_cast(input2_quantization_params.scale) / + twice_max_input_scale; + const double real_output_multiplier = + (twice_max_input_scale * twice_max_input_scale) / + static_cast((1 << data->arithmetic_params.left_shift * 2) * + output_quantization_params.scale); + QuantizeMultiplierSmallerThanOneExp( + real_input1_multiplier, &data->arithmetic_params.input1_multiplier, + &data->arithmetic_params.input1_shift); + QuantizeMultiplierSmallerThanOneExp( + real_input2_multiplier, &data->arithmetic_params.input2_multiplier, + &data->arithmetic_params.input2_shift); + QuantizeMultiplierSmallerThanOneExp( + real_output_multiplier, &data->arithmetic_params.output_multiplier, + &data->arithmetic_params.output_shift); + data->arithmetic_params.quantized_activation_min = + std::numeric_limits::min(); + data->arithmetic_params.quantized_activation_max = + std::numeric_limits::max(); + } + + data->requires_broadcast = !HaveSameShapes(input1, input2); + + micro_context->DeallocateTempTfLiteTensor(input1); + micro_context->DeallocateTempTfLiteTensor(input2); + micro_context->DeallocateTempTfLiteTensor(output); + return kTfLiteOk; +} + +inline int8_t SquaredDifference(int8_t x, int8_t y, + const ArithmeticParams& params) { + const int32_t input1_val = params.input1_offset + x; + const int32_t input2_val = params.input2_offset + y; + const int32_t shifted_input1_val = input1_val * (1 << params.left_shift); + const int32_t shifted_input2_val = input2_val * (1 << params.left_shift); + const int32_t scaled_input1_val = + MultiplyByQuantizedMultiplierSmallerThanOneExp( + shifted_input1_val, params.input1_multiplier, params.input1_shift); + const int32_t scaled_input2_val = + MultiplyByQuantizedMultiplierSmallerThanOneExp( + shifted_input2_val, params.input2_multiplier, params.input2_shift); + const int32_t raw_diff = scaled_input1_val - scaled_input2_val; + + // Max of this is 255^2 * (1 << 14), so won't overflow 32 bits. + const int32_t squared_raw_diff = raw_diff * raw_diff; + const int32_t raw_output = + MultiplyByQuantizedMultiplierSmallerThanOneExp( + squared_raw_diff, params.output_multiplier, params.output_shift) + + params.output_offset; + const int32_t clamped_output = + std::min(params.quantized_activation_max, + std::max(params.quantized_activation_min, raw_output)); + return static_cast(clamped_output); +} + +template +void EvalQuantizedSquaredDifference(TfLiteContext* context, TfLiteNode* node, + const OpData* data, + const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, + TfLiteEvalTensor* output) { + const auto* op_data = static_cast(node->user_data); + if (data->requires_broadcast) { + reference_integer_ops::BroadcastBinaryFunction4DSlow( + op_data->arithmetic_params, tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), + reference_integer_ops::CheckArithmeticParams, SquaredDifference); + } else { + const int flat_size = tflite::micro::GetTensorShape(input1).FlatSize(); + reference_integer_ops::ElementWise( + flat_size, op_data->arithmetic_params, + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorData(output), + reference_integer_ops::CheckArithmeticParams, SquaredDifference); + } +} + +template +void EvalSquaredDifference(TfLiteContext* context, TfLiteNode* node, + const OpData* data, const TfLiteEvalTensor* input1, + const TfLiteEvalTensor* input2, + TfLiteEvalTensor* output) { + if (data->requires_broadcast) { + reference_ops::BroadcastBinaryFunction4DSlow( + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), SquaredDifference); + } else { + reference_ops::BinaryFunction( + tflite::micro::GetTensorShape(input1), + tflite::micro::GetTensorData(input1), + tflite::micro::GetTensorShape(input2), + tflite::micro::GetTensorData(input2), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output), SquaredDifference); + } +} + +TfLiteStatus SquaredDifferenceEval(TfLiteContext* context, TfLiteNode* node) { + OpData* data = reinterpret_cast(node->user_data); + + const TfLiteEvalTensor* input1 = + tflite::micro::GetEvalInput(context, node, kInputTensor1); + const TfLiteEvalTensor* input2 = + tflite::micro::GetEvalInput(context, node, kInputTensor2); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + if (output->type == kTfLiteFloat32) { + EvalSquaredDifference(context, node, data, input1, input2, output); + } else if (output->type == kTfLiteInt32) { + EvalSquaredDifference(context, node, data, input1, input2, output); + } else if (output->type == kTfLiteInt8) { + EvalQuantizedSquaredDifference(context, node, data, input1, input2, + output); + } else { + MicroPrintf( + "SquaredDifference only supports FLOAT32, INT32 and INT8 now, got %d.", + output->type); + return kTfLiteError; + } + + return kTfLiteOk; +} +} // namespace + +TfLiteRegistration Register_SQUARED_DIFFERENCE() { + return tflite::micro::RegisterOp( + SquaredDifferenceInit, SquaredDifferencePrepare, SquaredDifferenceEval); +} + +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/squeeze.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/squeeze.cc index 0eb767db..e81b5b56 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/squeeze.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/squeeze.cc @@ -111,14 +111,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_SQUEEZE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/strided_slice.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/strided_slice.cc index d5b73b8f..832e2ccd 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/strided_slice.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/strided_slice.cc @@ -193,14 +193,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace strided_slice TfLiteRegistration Register_STRIDED_SLICE() { - return {/*init=*/strided_slice::Init, - /*free=*/nullptr, - /*prepare=*/strided_slice::Prepare, - /*invoke=*/strided_slice::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(strided_slice::Init, strided_slice::Prepare, + strided_slice::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/sub.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/sub.cc index de99149f..40bddbad 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/sub.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/sub.cc @@ -162,14 +162,7 @@ TfLiteStatus SubEval(TfLiteContext* context, TfLiteNode* node) { } TfLiteRegistration Register_SUB() { - return {/*init=*/SubInit, - /*free=*/nullptr, - /*prepare=*/SubPrepare, - /*invoke=*/SubEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(SubInit, SubPrepare, SubEval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/svdf.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/svdf.cc index f8a2bed2..5994db94 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/svdf.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/svdf.cc @@ -100,14 +100,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_SVDF() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/PrepareSvdf, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, PrepareSvdf, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/tanh.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/tanh.cc index a9f01c71..e97a9035 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/tanh.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/tanh.cc @@ -195,14 +195,8 @@ TfLiteStatus TanhEval(TfLiteContext* context, TfLiteNode* node) { } // namespace activations TfLiteRegistration Register_TANH() { - return {/*init=*/activations::TanhInit, - /*free=*/nullptr, - /*prepare=*/activations::TanhPrepare, - /*invoke=*/activations::TanhEval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp( + activations::TanhInit, activations::TanhPrepare, activations::TanhEval); } } // namespace micro } // namespace ops diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/transpose.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/transpose.cc index ba3d6e94..9f77e04d 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/transpose.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/transpose.cc @@ -116,13 +116,6 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_TRANSPOSE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/transpose_conv.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/transpose_conv.cc index dcf007c5..0b2afd5b 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/transpose_conv.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/transpose_conv.cc @@ -266,7 +266,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorShape(filter), tflite::micro::GetTensorData(filter), tflite::micro::GetTensorShape(bias), - tflite::micro::GetTensorData(bias), + tflite::micro::GetOptionalTensorData(bias), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output), tflite::micro::GetTensorShape(nullptr), nullptr); @@ -282,7 +282,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorShape(filter), tflite::micro::GetTensorData(filter), tflite::micro::GetTensorShape(bias), - tflite::micro::GetTensorData(bias), + tflite::micro::GetOptionalTensorData(bias), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output), tflite::micro::GetTensorShape(nullptr), nullptr, scratch_buffer); @@ -293,7 +293,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { context->GetScratchBuffer(context, data.scratch_buffer_index)); // TODO(b/192090531): Remove this once all 8x16 transpose conv models use // 64-bit biases. - if (bias->type == kTfLiteInt16) { + if (bias != nullptr && bias->type == kTfLiteInt16) { std::int64_t* bias_converted_buffer = static_cast(context->GetScratchBuffer( context, data.bias_converted_buffer_index)); @@ -319,7 +319,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetTensorShape(filter), tflite::micro::GetTensorData(filter), tflite::micro::GetTensorShape(bias), - tflite::micro::GetTensorData(bias), + tflite::micro::GetOptionalTensorData(bias), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output), tflite::micro::GetTensorShape(nullptr), nullptr, scratch_buffer); @@ -337,14 +337,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_TRANSPOSE_CONV() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/unidirectional_sequence_lstm.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/unidirectional_sequence_lstm.cc new file mode 100644 index 00000000..7f3c50e4 --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/unidirectional_sequence_lstm.cc @@ -0,0 +1,1696 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/lstm_eval.h" +#include "tensorflow/lite/micro/kernels/lstm_shared.h" +#include "tensorflow/lite/micro/kernels/micro_tensor_utils.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" + +namespace tflite { + +namespace { + +constexpr int scratch_index_size = 12; + +struct UnidirectionalSequenceLstmOpData { + // If the lstm is layer norm. + bool use_layer_norm; + // The scratch index. + int scratch_index[scratch_index_size]; + + int32_t row_sums_size; + int32_t* row_sums; + bool compute_row_sums = false; + + int32_t input_zero_point; + int32_t output_state_zero_point; + + IntegerLstmParameter integer_lstm_param; + HybridLstmScales hybrid_lstm_scales; +}; + +TfLiteStatus PopulateQuantizedLstmParams8x8_16( + TfLiteContext* context, TfLiteNode* node, + IntegerLstmParameter* integer_lstm_param) { + MicroContext* micro_context = GetMicroContext(context); + + // Calculate quantized clip for projection and cell. + const auto* params = + static_cast(node->builtin_data); + const float cell_clip = params->cell_clip; + const float proj_clip = params->proj_clip; + + TfLiteTensor* cell_state = + micro_context->AllocateTempInputTensor(node, kLstmCellStateTensor); + TF_LITE_ENSURE(context, cell_state != nullptr); + TF_LITE_ENSURE(context, cell_state->is_variable); + TfLiteTensor* output_tensor = + micro_context->AllocateTempOutputTensor(node, kLstmOutputTensor); + + TF_LITE_ENSURE(context, + cell_state->quantization.type != kTfLiteNoQuantization); + auto* cell_state_params = + static_cast(cell_state->quantization.params); + TF_LITE_ENSURE(context, + output_tensor->quantization.type != kTfLiteNoQuantization); + auto* proj_params = static_cast( + output_tensor->quantization.params); + if (cell_clip > 0.0f) { + integer_lstm_param->quantized_cell_clip = static_cast(std::min( + std::max(cell_clip / cell_state_params->scale->data[0], -32768.0f), + 32767.0f)); + } else { + integer_lstm_param->quantized_cell_clip = 0; + } + if (proj_clip > 0.0f) { + integer_lstm_param->quantized_proj_clip = static_cast(std::min( + std::max(proj_clip / proj_params->scale->data[0], -128.0f), 127.0f)); + } else { + integer_lstm_param->quantized_proj_clip = 0; + } + + // Calculate effective scales. + UnidirectionalSequenceLstmOpData* op_data = + static_cast(node->user_data); + const bool use_layer_norm = op_data->use_layer_norm; + + TfLiteTensor* input = + micro_context->AllocateTempInputTensor(node, kLstmInputTensor); + + TfLiteTensor* input_to_input_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToInputWeightsTensor); + TfLiteTensor* input_to_forget_weights = + micro_context->AllocateTempInputTensor(node, + kLstmInputToForgetWeightsTensor); + TfLiteTensor* input_to_cell_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToCellWeightsTensor); + TfLiteTensor* input_to_output_weights = + micro_context->AllocateTempInputTensor(node, + kLstmInputToOutputWeightsTensor); + + TfLiteTensor* recurrent_to_input_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToInputWeightsTensor); + TfLiteTensor* recurrent_to_forget_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToForgetWeightsTensor); + TfLiteTensor* recurrent_to_cell_weights = + micro_context->AllocateTempInputTensor(node, + kLstmRecurrentToCellWeightsTensor); + TfLiteTensor* recurrent_to_output_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToOutputWeightsTensor); + + TfLiteTensor* cell_to_input_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToInputWeightsTensor); + TfLiteTensor* cell_to_forget_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToForgetWeightsTensor); + TfLiteTensor* cell_to_output_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToOutputWeightsTensor); + + TfLiteTensor* input_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmInputLayerNormCoefficientsTensor); + TfLiteTensor* forget_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmForgetLayerNormCoefficientsTensor); + TfLiteTensor* cell_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmCellLayerNormCoefficientsTensor); + TfLiteTensor* output_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmOutputLayerNormCoefficientsTensor); + + TfLiteTensor* projection_weights = micro_context->AllocateTempInputTensor( + node, kLstmProjectionWeightsTensor); + + TfLiteTensor* output_state = + micro_context->AllocateTempInputTensor(node, kLstmOutputStateTensor); + TF_LITE_ENSURE(context, output_state != nullptr); + TF_LITE_ENSURE(context, output_state->is_variable); + + // Since we have already checked that weights are all there or none, we can + // check the existence of only one to get the condition. + const bool use_cifg = (input_to_input_weights == nullptr); + const bool use_peephole = (cell_to_output_weights != nullptr); + const bool use_projection = (projection_weights != nullptr); + + // Get intermediate scales and zero points. + float intermediate_scale[5]; + int32_t intermediate_zp[5]; + for (int i = 0; i < 4; ++i) { + if (use_layer_norm) { + TfLiteTensor* intermediate = + micro_context->AllocateTempIntermediateTensor(node, i); + TF_LITE_ENSURE(context, + intermediate->quantization.type != kTfLiteNoQuantization); + auto* params_intermediate = static_cast( + intermediate->quantization.params); + intermediate_scale[i] = params_intermediate->scale->data[0]; + intermediate_zp[i] = params_intermediate->zero_point->data[0]; + if (intermediate != nullptr) { + micro_context->DeallocateTempTfLiteTensor(intermediate); + } + } else { + // Q3.12 for activation functions. + intermediate_scale[i] = std::pow(2.0f, -12.0f); + intermediate_zp[i] = 0; + } + } + // In the absence of projection, hidden becomes otuput and this intermediate + // is ignored. + TfLiteTensor* hidden = micro_context->AllocateTempIntermediateTensor(node, 4); + TF_LITE_ENSURE(context, hidden->quantization.type != kTfLiteNoQuantization); + auto* hidden_params = + static_cast(hidden->quantization.params); + intermediate_scale[4] = hidden_params->scale->data[0]; + intermediate_zp[4] = hidden_params->zero_point->data[0]; + if (hidden != nullptr) { + micro_context->DeallocateTempTfLiteTensor(hidden); + } + + // Scales. + const float default_scale = 1.0; + float input_scale = default_scale; + float input_to_input_weight_scale = default_scale; + float recurrent_to_input_weight_scale = default_scale; + float cell_to_input_weight_scale = default_scale; + float input_to_forget_weight_scale = default_scale; + float recurrent_to_forget_weight_scale = default_scale; + float cell_to_forget_weight_scale = default_scale; + float input_to_cell_weight_scale = default_scale; + float recurrent_to_cell_weight_scale = default_scale; + float input_to_output_weight_scale = default_scale; + float recurrent_to_output_weight_scale = default_scale; + float cell_to_output_weight_scale = default_scale; + float projection_weight_scale = default_scale; + float layer_norm_input_scale = default_scale; + float layer_norm_forget_scale = default_scale; + float layer_norm_cell_scale = default_scale; + float layer_norm_output_scale = default_scale; + float output_state_scale = default_scale; + int cell_scale = 1; + + // Effective scales. + float effective_input_to_input_scale = default_scale; + float effective_recurrent_to_input_scale = default_scale; + float effective_cell_to_input_scale = default_scale; + float effective_input_to_forget_scale = default_scale; + float effective_recurrent_to_forget_scale = default_scale; + float effective_cell_to_forget_scale = default_scale; + float effective_input_to_cell_scale = default_scale; + float effective_recurrent_to_cell_scale = default_scale; + float effective_input_to_output_scale = default_scale; + float effective_recurrent_to_output_scale = default_scale; + float effective_cell_to_output_scale = default_scale; + float effective_proj_scale = default_scale; + float effective_hidden_scale = default_scale; + + // Populate scales. + if (!use_cifg) { + input_to_input_weight_scale = input_to_input_weights->params.scale; + recurrent_to_input_weight_scale = recurrent_to_input_weights->params.scale; + } + + if (use_peephole) { + if (!use_cifg) { + cell_to_input_weight_scale = cell_to_input_weights->params.scale; + } + cell_to_forget_weight_scale = cell_to_forget_weights->params.scale; + cell_to_output_weight_scale = cell_to_output_weights->params.scale; + } + + if (use_layer_norm) { + if (!use_cifg) { + layer_norm_input_scale = input_layer_norm_coefficients->params.scale; + } + layer_norm_forget_scale = forget_layer_norm_coefficients->params.scale; + layer_norm_cell_scale = cell_layer_norm_coefficients->params.scale; + layer_norm_output_scale = output_layer_norm_coefficients->params.scale; + } + + if (use_projection) { + projection_weight_scale = projection_weights->params.scale; + } + output_state_scale = output_state->params.scale; + + input_to_forget_weight_scale = input_to_forget_weights->params.scale; + input_to_cell_weight_scale = input_to_cell_weights->params.scale; + input_to_output_weight_scale = input_to_output_weights->params.scale; + recurrent_to_forget_weight_scale = recurrent_to_forget_weights->params.scale; + recurrent_to_cell_weight_scale = recurrent_to_cell_weights->params.scale; + recurrent_to_output_weight_scale = recurrent_to_output_weights->params.scale; + + // Check cell state (already used above) + TF_LITE_ENSURE(context, CheckedLog2(cell_state->params.scale, &cell_scale)); + // TF_LITE_ENSURE(context, cell_scale <= -9); + integer_lstm_param->cell_scale = cell_scale; + input_scale = input->params.scale; + + // Calculate effective scales. + if (!use_cifg) { + effective_input_to_input_scale = + input_to_input_weight_scale * input_scale / intermediate_scale[0]; + effective_recurrent_to_input_scale = recurrent_to_input_weight_scale * + output_state_scale / + intermediate_scale[0]; + } + effective_input_to_forget_scale = + input_to_forget_weight_scale * input_scale / intermediate_scale[1]; + effective_recurrent_to_forget_scale = recurrent_to_forget_weight_scale * + output_state_scale / + intermediate_scale[1]; + + effective_input_to_cell_scale = + input_to_cell_weight_scale * input_scale / intermediate_scale[2]; + effective_recurrent_to_cell_scale = recurrent_to_cell_weight_scale * + output_state_scale / + intermediate_scale[2]; + + effective_input_to_output_scale = + input_to_output_weight_scale * input_scale / intermediate_scale[3]; + effective_recurrent_to_output_scale = recurrent_to_output_weight_scale * + output_state_scale / + intermediate_scale[3]; + + effective_hidden_scale = + std::pow(2.0f, -15.0f) / intermediate_scale[4] * std::pow(2.0f, -15.0f); + + effective_proj_scale = + projection_weight_scale * intermediate_scale[4] / output_state_scale; + + if (use_peephole) { + if (!use_cifg) { + effective_cell_to_input_scale = + std::pow(2.0f, static_cast(cell_scale)) * + cell_to_input_weight_scale / intermediate_scale[0]; + } + effective_cell_to_forget_scale = + std::pow(2.0f, static_cast(cell_scale)) * + cell_to_forget_weight_scale / intermediate_scale[1]; + effective_cell_to_output_scale = + std::pow(2.0f, static_cast(cell_scale)) * + cell_to_output_weight_scale / intermediate_scale[3]; + } + + // Decompose scales. + int shift_output; + QuantizeMultiplier(static_cast(effective_input_to_input_scale), + &integer_lstm_param->effective_input_to_input_scale_a, + &shift_output); + integer_lstm_param->effective_input_to_input_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_recurrent_to_input_scale), + &integer_lstm_param->effective_recurrent_to_input_scale_a, + &shift_output); + integer_lstm_param->effective_recurrent_to_input_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_cell_to_input_scale), + &integer_lstm_param->effective_cell_to_input_scale_a, + &shift_output); + integer_lstm_param->effective_cell_to_input_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_input_to_forget_scale), + &integer_lstm_param->effective_input_to_forget_scale_a, + &shift_output); + integer_lstm_param->effective_input_to_forget_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_recurrent_to_forget_scale), + &integer_lstm_param->effective_recurrent_to_forget_scale_a, + &shift_output); + integer_lstm_param->effective_recurrent_to_forget_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_cell_to_forget_scale), + &integer_lstm_param->effective_cell_to_forget_scale_a, + &shift_output); + integer_lstm_param->effective_cell_to_forget_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_input_to_cell_scale), + &integer_lstm_param->effective_input_to_cell_scale_a, + &shift_output); + integer_lstm_param->effective_input_to_cell_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_recurrent_to_cell_scale), + &integer_lstm_param->effective_recurrent_to_cell_scale_a, + &shift_output); + integer_lstm_param->effective_recurrent_to_cell_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_input_to_output_scale), + &integer_lstm_param->effective_input_to_output_scale_a, + &shift_output); + integer_lstm_param->effective_input_to_output_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_recurrent_to_output_scale), + &integer_lstm_param->effective_recurrent_to_output_scale_a, + &shift_output); + integer_lstm_param->effective_recurrent_to_output_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_cell_to_output_scale), + &integer_lstm_param->effective_cell_to_output_scale_a, + &shift_output); + integer_lstm_param->effective_cell_to_output_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_proj_scale), + &integer_lstm_param->effective_proj_scale_a, + &shift_output); + integer_lstm_param->effective_proj_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(effective_hidden_scale), + &integer_lstm_param->effective_hidden_scale_a, + &shift_output); + integer_lstm_param->effective_hidden_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(layer_norm_input_scale), + &integer_lstm_param->layer_norm_input_scale_a, + &shift_output); + integer_lstm_param->layer_norm_input_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(layer_norm_forget_scale), + &integer_lstm_param->layer_norm_forget_scale_a, + &shift_output); + integer_lstm_param->layer_norm_forget_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(layer_norm_cell_scale), + &integer_lstm_param->layer_norm_cell_scale_a, + &shift_output); + integer_lstm_param->layer_norm_cell_scale_b = + static_cast(shift_output); + QuantizeMultiplier(static_cast(layer_norm_output_scale), + &integer_lstm_param->layer_norm_output_scale_a, + &shift_output); + integer_lstm_param->layer_norm_output_scale_b = + static_cast(shift_output); + + integer_lstm_param->hidden_zp = intermediate_zp[4]; + + // 10000 is used to make sure the kernel logic does not overflow. + if (!use_cifg) { + integer_lstm_param->input_variance_guard = + std::max(1, static_cast(10000 * layer_norm_input_scale)); + } + integer_lstm_param->forget_variance_guard = + std::max(1, static_cast(10000 * layer_norm_forget_scale)); + integer_lstm_param->cell_variance_guard = + std::max(1, static_cast(10000 * layer_norm_cell_scale)); + integer_lstm_param->output_variance_guard = + std::max(1, static_cast(10000 * layer_norm_output_scale)); + + if (cell_state != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_state); + } + if (output_tensor != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output_tensor); + } + if (input != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input); + } + if (input_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_input_weights); + } + if (input_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_forget_weights); + } + if (input_to_cell_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_cell_weights); + } + if (input_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_output_weights); + } + if (recurrent_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_input_weights); + } + if (recurrent_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_forget_weights); + } + if (recurrent_to_cell_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_cell_weights); + } + if (recurrent_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_output_weights); + } + if (cell_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_input_weights); + } + if (cell_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_forget_weights); + } + if (cell_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_output_weights); + } + if (input_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_layer_norm_coefficients); + } + if (forget_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(forget_layer_norm_coefficients); + } + if (cell_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_layer_norm_coefficients); + } + if (output_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output_layer_norm_coefficients); + } + if (projection_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(projection_weights); + } + if (output_state != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output_state); + } + + return kTfLiteOk; +} + +// Temporary buffers used for hybrid mode +enum HybridTempBuffer { + kPrimaryScratchBuffer = 0, + kInputQuantized = 1, + kOutputStateQuantized = 2, + kCellStateQuantized = 3, + kInputScalingFactors = 4, + kOutputStateScalingFactors = 5, + kProductScalingFactors = 6, + kRecoveredCellWeights = 7, + kAccumScratch = 8, + kInputZeroPoints = 9, + kOutputStateZeroPoints = 10, + kScales = 11, + kNumHybridTempBuffers = 12, +}; + +void* UnidirectionalSequenceLstmInit(TfLiteContext* context, const char* buffer, + size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + return context->AllocatePersistentBuffer( + context, sizeof(UnidirectionalSequenceLstmOpData)); +} + +// Check that input tensor dimensions matches with each other. +TfLiteStatus SetHybridScales(TfLiteContext* context, TfLiteNode* node) { + UnidirectionalSequenceLstmOpData* op_data = + reinterpret_cast(node->user_data); + MicroContext* micro_context = GetMicroContext(context); + + TfLiteTensor* input_to_input_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToInputWeightsTensor); + op_data->hybrid_lstm_scales.input_to_input_weights_scale = + (input_to_input_weights != nullptr) ? input_to_input_weights->params.scale + : 1.0f; + + TfLiteTensor* input_to_forget_weights = + micro_context->AllocateTempInputTensor(node, + kLstmInputToForgetWeightsTensor); + op_data->hybrid_lstm_scales.input_to_forget_weights_scale = + (input_to_forget_weights != nullptr) + ? input_to_forget_weights->params.scale + : 1.0f; + + TfLiteTensor* input_to_cell_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToCellWeightsTensor); + op_data->hybrid_lstm_scales.input_to_cell_weights_scale = + (input_to_cell_weights != nullptr) ? input_to_cell_weights->params.scale + : 1.0f; + + TfLiteTensor* input_to_output_weights = + micro_context->AllocateTempInputTensor(node, + kLstmInputToOutputWeightsTensor); + op_data->hybrid_lstm_scales.input_to_output_weights_scale = + (input_to_output_weights != nullptr) + ? input_to_output_weights->params.scale + : 1.0f; + + op_data->hybrid_lstm_scales.aux_input_to_input_weights_scale = 1.0f; + op_data->hybrid_lstm_scales.aux_input_to_forget_weights_scale = 1.0f; + op_data->hybrid_lstm_scales.aux_input_to_cell_weights_scale = 1.0f; + op_data->hybrid_lstm_scales.aux_input_to_output_weights_scale = 1.0f; + + TfLiteTensor* recurrent_to_input_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToInputWeightsTensor); + op_data->hybrid_lstm_scales.recurrent_to_input_weights_scale = + (recurrent_to_input_weights != nullptr) + ? recurrent_to_input_weights->params.scale + : 1.0f; + + TfLiteTensor* recurrent_to_forget_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToForgetWeightsTensor); + op_data->hybrid_lstm_scales.recurrent_to_forget_weights_scale = + (recurrent_to_forget_weights != nullptr) + ? recurrent_to_forget_weights->params.scale + : 1.0f; + + TfLiteTensor* recurrent_to_cell_weights = + micro_context->AllocateTempInputTensor(node, + kLstmRecurrentToCellWeightsTensor); + op_data->hybrid_lstm_scales.recurrent_to_cell_weights_scale = + (recurrent_to_cell_weights != nullptr) + ? recurrent_to_cell_weights->params.scale + : 1.0f; + + TfLiteTensor* recurrent_to_output_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToOutputWeightsTensor); + op_data->hybrid_lstm_scales.recurrent_to_output_weights_scale = + (recurrent_to_output_weights != nullptr) + ? recurrent_to_output_weights->params.scale + : 1.0f; + + TfLiteTensor* cell_to_input_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToInputWeightsTensor); + op_data->hybrid_lstm_scales.cell_to_input_weights_scale = + (cell_to_input_weights != nullptr) ? cell_to_input_weights->params.scale + : 1.0f; + + TfLiteTensor* cell_to_forget_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToForgetWeightsTensor); + op_data->hybrid_lstm_scales.cell_to_forget_weights_scale = + (cell_to_forget_weights != nullptr) ? cell_to_forget_weights->params.scale + : 1.0f; + + TfLiteTensor* cell_to_output_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToOutputWeightsTensor); + op_data->hybrid_lstm_scales.cell_to_output_weights_scale = + (cell_to_output_weights != nullptr) ? cell_to_output_weights->params.scale + : 1.0f; + + TfLiteTensor* projection_weights = micro_context->AllocateTempInputTensor( + node, kLstmProjectionWeightsTensor); + op_data->hybrid_lstm_scales.projection_weights_scale = + (projection_weights != nullptr) ? projection_weights->params.scale : 1.0f; + + if (input_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_input_weights); + } + + if (input_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_forget_weights); + } + + if (input_to_cell_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_cell_weights); + } + + if (input_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_output_weights); + } + + if (recurrent_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_input_weights); + } + + if (recurrent_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_forget_weights); + } + + if (recurrent_to_cell_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_cell_weights); + } + + if (recurrent_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_output_weights); + } + + if (cell_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_input_weights); + } + + if (cell_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_forget_weights); + } + + if (cell_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_output_weights); + } + + if (projection_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(projection_weights); + } + + return kTfLiteOk; +} + +// Check that input tensor dimensions matches with each other. +TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context, + TfLiteNode* node, int n_input, + int n_output, int n_cell, + bool use_layer_norm, bool is_integer) { + MicroContext* micro_context = GetMicroContext(context); + + const auto* params = reinterpret_cast(node->builtin_data); + + // Making sure clipping parameters have valid values. + // == 0 means no clipping + // > 0 means clipping + TF_LITE_ENSURE(context, params->cell_clip >= 0); + TF_LITE_ENSURE(context, params->proj_clip >= 0); + + TfLiteTensor* input_to_input_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToInputWeightsTensor); + if (input_to_input_weights != nullptr) { + TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->data[0], n_cell); + TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->data[1], n_input); + } + + TfLiteTensor* input_to_forget_weights = + micro_context->AllocateTempInputTensor(node, + kLstmInputToForgetWeightsTensor); + TF_LITE_ENSURE_EQ(context, input_to_forget_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, input_to_forget_weights->dims->data[0], n_cell); + TF_LITE_ENSURE_EQ(context, input_to_forget_weights->dims->data[1], n_input); + + TfLiteTensor* input_to_cell_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToCellWeightsTensor); + TF_LITE_ENSURE_EQ(context, input_to_cell_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, input_to_cell_weights->dims->data[0], n_cell); + TF_LITE_ENSURE_EQ(context, input_to_cell_weights->dims->data[1], n_input); + + TfLiteTensor* recurrent_to_input_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToInputWeightsTensor); + if (recurrent_to_input_weights != nullptr) { + TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights->dims->data[0], + n_cell); + TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights->dims->data[1], + n_output); + } + + TfLiteTensor* recurrent_to_forget_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToForgetWeightsTensor); + TF_LITE_ENSURE_EQ(context, recurrent_to_forget_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, recurrent_to_forget_weights->dims->data[0], + n_cell); + TF_LITE_ENSURE_EQ(context, recurrent_to_forget_weights->dims->data[1], + n_output); + + TfLiteTensor* recurrent_to_cell_weights = + micro_context->AllocateTempInputTensor(node, + kLstmRecurrentToCellWeightsTensor); + TF_LITE_ENSURE_EQ(context, recurrent_to_cell_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, recurrent_to_cell_weights->dims->data[0], n_cell); + TF_LITE_ENSURE_EQ(context, recurrent_to_cell_weights->dims->data[1], + n_output); + + // We make sure the input-gate's parameters are either both present (regular + // LSTM) or not at all (CIFG-LSTM). + const bool cifg_weights_all_or_none = + ((input_to_input_weights != nullptr) && + (recurrent_to_input_weights != nullptr)) || + ((input_to_input_weights == nullptr) && + (recurrent_to_input_weights == nullptr)); + TF_LITE_ENSURE(context, cifg_weights_all_or_none == true); + + TfLiteTensor* cell_to_input_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToInputWeightsTensor); + if (cell_to_input_weights != nullptr) { + TF_LITE_ENSURE_EQ(context, cell_to_input_weights->dims->size, 1); + TF_LITE_ENSURE_EQ(context, cell_to_input_weights->dims->data[0], n_cell); + TF_LITE_ENSURE_TYPES_EQ( + context, cell_to_input_weights->type, + is_integer ? kTfLiteInt16 : input_to_forget_weights->type); + } + + TfLiteTensor* cell_to_forget_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToForgetWeightsTensor); + if (cell_to_forget_weights != nullptr) { + TF_LITE_ENSURE_EQ(context, cell_to_forget_weights->dims->size, 1); + TF_LITE_ENSURE_EQ(context, cell_to_forget_weights->dims->data[0], n_cell); + TF_LITE_ENSURE_TYPES_EQ( + context, cell_to_forget_weights->type, + is_integer ? kTfLiteInt16 : input_to_forget_weights->type); + } + + TfLiteTensor* cell_to_output_weights = micro_context->AllocateTempInputTensor( + node, kLstmCellToOutputWeightsTensor); + if (cell_to_output_weights != nullptr) { + TF_LITE_ENSURE_EQ(context, cell_to_output_weights->dims->size, 1); + TF_LITE_ENSURE_EQ(context, cell_to_output_weights->dims->data[0], n_cell); + TF_LITE_ENSURE_TYPES_EQ( + context, cell_to_output_weights->type, + is_integer ? kTfLiteInt16 : input_to_forget_weights->type); + } + + // Making sure the peephole weights are there all or none. + const bool use_cifg = (input_to_input_weights == nullptr); + const bool peephole_weights_all_or_none = + ((cell_to_input_weights != nullptr || use_cifg) && + (cell_to_forget_weights != nullptr) && + (cell_to_output_weights != nullptr)) || + ((cell_to_input_weights == nullptr) && + (cell_to_forget_weights == nullptr) && + (cell_to_output_weights == nullptr)); + TF_LITE_ENSURE(context, peephole_weights_all_or_none == true); + + // Make sure the input gate bias is present only when not a CIFG-LSTM. + TfLiteTensor* input_gate_bias = + micro_context->AllocateTempInputTensor(node, kLstmInputGateBiasTensor); + if (use_cifg) { + TF_LITE_ENSURE_EQ(context, input_gate_bias, nullptr); + } else { + TF_LITE_ENSURE_EQ(context, input_gate_bias->dims->size, 1); + TF_LITE_ENSURE_EQ(context, input_gate_bias->dims->data[0], n_cell); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, input_gate_bias->type, kTfLiteInt32); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, input_gate_bias->type, kTfLiteFloat32); + } + } + + TfLiteTensor* forget_gate_bias = + micro_context->AllocateTempInputTensor(node, kLstmForgetGateBiasTensor); + TF_LITE_ENSURE_EQ(context, forget_gate_bias->dims->size, 1); + TF_LITE_ENSURE_EQ(context, forget_gate_bias->dims->data[0], n_cell); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, forget_gate_bias->type, kTfLiteInt32); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, forget_gate_bias->type, kTfLiteFloat32); + } + + TfLiteTensor* cell_gate_bias = + micro_context->AllocateTempInputTensor(node, kLstmCellGateBiasTensor); + TF_LITE_ENSURE_EQ(context, cell_gate_bias->dims->size, 1); + TF_LITE_ENSURE_EQ(context, cell_gate_bias->dims->data[0], n_cell); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, cell_gate_bias->type, kTfLiteInt32); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, cell_gate_bias->type, kTfLiteFloat32); + } + + TfLiteTensor* output_gate_bias = + micro_context->AllocateTempInputTensor(node, kLstmOutputGateBiasTensor); + TF_LITE_ENSURE_EQ(context, output_gate_bias->dims->size, 1); + TF_LITE_ENSURE_EQ(context, output_gate_bias->dims->data[0], n_cell); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, output_gate_bias->type, kTfLiteInt32); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, output_gate_bias->type, kTfLiteFloat32); + } + + TfLiteTensor* projection_weights = micro_context->AllocateTempInputTensor( + node, kLstmProjectionWeightsTensor); + if (projection_weights != nullptr) { + TF_LITE_ENSURE_EQ(context, projection_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, projection_weights->dims->data[0], n_output); + TF_LITE_ENSURE_EQ(context, projection_weights->dims->data[1], n_cell); + } + + TfLiteTensor* projection_bias = + micro_context->AllocateTempInputTensor(node, kLstmProjectionBiasTensor); + if (projection_bias != nullptr) { + TF_LITE_ENSURE_EQ(context, projection_bias->dims->size, 1); + TF_LITE_ENSURE_EQ(context, projection_bias->dims->data[0], n_output); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, projection_bias->type, kTfLiteInt32); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, projection_bias->type, kTfLiteFloat32); + } + } + + // Making sure the projection tensors are consistent: + // 1) If projection weight is not present, then projection bias should not be + // present. + // 2) If projection weight is present, then projection bias is optional. + const bool projecton_tensors_consistent = + ((projection_weights != nullptr) || (projection_bias == nullptr)); + TF_LITE_ENSURE(context, projecton_tensors_consistent == true); + + if (use_layer_norm) { + TfLiteTensor* input_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmInputLayerNormCoefficientsTensor); + if (use_cifg) { + TF_LITE_ENSURE_EQ(context, input_layer_norm_coefficients, nullptr); + } else { + TF_LITE_ENSURE(context, input_layer_norm_coefficients != nullptr); + TF_LITE_ENSURE_EQ(context, input_layer_norm_coefficients->dims->size, 1); + TF_LITE_ENSURE_EQ(context, input_layer_norm_coefficients->dims->data[0], + n_cell); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, input_layer_norm_coefficients->type, + kTfLiteInt16); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, input_layer_norm_coefficients->type, + kTfLiteFloat32); + } + } + + TfLiteTensor* forget_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmForgetLayerNormCoefficientsTensor); + TF_LITE_ENSURE_EQ(context, forget_layer_norm_coefficients->dims->size, 1); + TF_LITE_ENSURE_EQ(context, forget_layer_norm_coefficients->dims->data[0], + n_cell); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, forget_layer_norm_coefficients->type, + kTfLiteInt16); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, forget_layer_norm_coefficients->type, + kTfLiteFloat32); + } + + TfLiteTensor* cell_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmCellLayerNormCoefficientsTensor); + TF_LITE_ENSURE_EQ(context, cell_layer_norm_coefficients->dims->size, 1); + TF_LITE_ENSURE_EQ(context, cell_layer_norm_coefficients->dims->data[0], + n_cell); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, cell_layer_norm_coefficients->type, + kTfLiteInt16); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, cell_layer_norm_coefficients->type, + kTfLiteFloat32); + } + + TfLiteTensor* output_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmOutputLayerNormCoefficientsTensor); + TF_LITE_ENSURE_EQ(context, output_layer_norm_coefficients->dims->size, 1); + TF_LITE_ENSURE_EQ(context, output_layer_norm_coefficients->dims->data[0], + n_cell); + if (is_integer) { + TF_LITE_ENSURE_TYPES_EQ(context, output_layer_norm_coefficients->type, + kTfLiteInt16); + } else { + TF_LITE_ENSURE_TYPES_EQ(context, output_layer_norm_coefficients->type, + kTfLiteFloat32); + } + if (input_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_layer_norm_coefficients); + } + if (forget_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(forget_layer_norm_coefficients); + } + if (cell_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_layer_norm_coefficients); + } + if (output_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output_layer_norm_coefficients); + } + } + + if (input_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_input_weights); + } + if (input_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_forget_weights); + } + if (input_to_cell_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_cell_weights); + } + if (recurrent_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_input_weights); + } + if (recurrent_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_forget_weights); + } + micro_context->DeallocateTempTfLiteTensor(recurrent_to_cell_weights); + if (cell_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_input_weights); + } + if (cell_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_forget_weights); + } + if (cell_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_to_output_weights); + } + if (input_gate_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_gate_bias); + } + if (forget_gate_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(forget_gate_bias); + } + if (cell_gate_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_gate_bias); + } + if (output_gate_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output_gate_bias); + } + if (projection_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(projection_weights); + } + if (projection_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(projection_bias); + } + + return kTfLiteOk; +} + +TfLiteStatus PrecomputeZeroPointTimesWeightWithBias( + TfLiteContext* context, int32_t zero_point, + const TfLiteTensor* weight_tensor, const TfLiteTensor* bias_tensor, + int32_t** output) { + if (weight_tensor == nullptr) { + return kTfLiteOk; + } + + const RuntimeShape& weight_shape = GetTensorShape(weight_tensor); + TF_LITE_ENSURE_EQ(context, weight_shape.DimensionsCount(), 2); + const int row = weight_shape.Dims(0); + const int col = weight_shape.Dims(1); + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + *output = static_cast( + context->AllocatePersistentBuffer(context, row * sizeof(int32_t))); + + if (bias_tensor == nullptr) { + memset(*output, 0, row * sizeof(int32_t)); + } else { + const int32_t* bias = GetTensorData(bias_tensor); + memcpy(*output, bias, row * sizeof(int32_t)); + } + if (zero_point != 0) { + const int8_t* weight = GetTensorData(weight_tensor); + micro_tensor_utils::MatrixScalarMultiplyAccumulate(weight, zero_point, row, + col, *output); + } + return kTfLiteOk; +} + +TfLiteStatus PopulatePrecomputedZPTimesWeightsWithBias( + TfLiteContext* context, UnidirectionalSequenceLstmOpData* op_data, + TfLiteNode* node) { + MicroContext* micro_context = GetMicroContext(context); + + TfLiteTensor* input = + micro_context->AllocateTempInputTensor(node, kLstmInputTensor); + TfLiteTensor* output_state = + micro_context->AllocateTempInputTensor(node, kLstmOutputStateTensor); + TF_LITE_ENSURE(context, output_state != nullptr); + TF_LITE_ENSURE(context, output_state->is_variable); + + const int32_t input_zero_point = -input->params.zero_point; + const int32_t output_state_zero_point = -output_state->params.zero_point; + + TfLiteTensor* input_to_input_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToInputWeightsTensor); + TfLiteTensor* input_to_forget_weights = + micro_context->AllocateTempInputTensor(node, + kLstmInputToForgetWeightsTensor); + TfLiteTensor* input_to_cell_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToCellWeightsTensor); + TfLiteTensor* input_to_output_weights = + micro_context->AllocateTempInputTensor(node, + kLstmInputToOutputWeightsTensor); + + TfLiteTensor* recurrent_to_input_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToInputWeightsTensor); + TfLiteTensor* recurrent_to_forget_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToForgetWeightsTensor); + TfLiteTensor* recurrent_to_cell_weights = + micro_context->AllocateTempInputTensor(node, + kLstmRecurrentToCellWeightsTensor); + TfLiteTensor* recurrent_to_output_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToOutputWeightsTensor); + + TfLiteTensor* projection_weights = micro_context->AllocateTempInputTensor( + node, kLstmProjectionWeightsTensor); + TfLiteTensor* projection_bias = + micro_context->AllocateTempInputTensor(node, kLstmProjectionBiasTensor); + + IntegerLstmParameter* integer_lstm_params = &op_data->integer_lstm_param; + + TfLiteTensor* intermediate = + micro_context->AllocateTempIntermediateTensor(node, 4); + TF_LITE_ENSURE(context, + intermediate->quantization.type != kTfLiteNoQuantization); + const auto* params = + static_cast(intermediate->quantization.params); + const int32_t hidden_zp = params->zero_point->data[0]; + + // Get bias and perform zero point calculation. + // When there is layer normalization, the gate bias does not apply to matmul + // directly: + // y = ln(w * x + w * r + w * c) + b. + const bool is_layer_norm = op_data->use_layer_norm; + + // Forget gate. + TfLiteTensor* forget_gate_bias = is_layer_norm + ? nullptr + : micro_context->AllocateTempInputTensor( + node, kLstmForgetGateBiasTensor); + TF_LITE_ENSURE_OK( + context, + PrecomputeZeroPointTimesWeightWithBias( + context, input_zero_point, input_to_forget_weights, forget_gate_bias, + &(integer_lstm_params->input_to_forget_effective_bias))); + + TF_LITE_ENSURE_OK( + context, + PrecomputeZeroPointTimesWeightWithBias( + context, output_state_zero_point, recurrent_to_forget_weights, + nullptr, &(integer_lstm_params->recurrent_to_forget_effective_bias))); + + // Modulation gate. + TfLiteTensor* cell_gate_bias = is_layer_norm + ? nullptr + : micro_context->AllocateTempInputTensor( + node, kLstmCellGateBiasTensor); + TF_LITE_ENSURE_OK( + context, + PrecomputeZeroPointTimesWeightWithBias( + context, input_zero_point, input_to_cell_weights, cell_gate_bias, + &(integer_lstm_params->input_to_cell_effective_bias))); + TF_LITE_ENSURE_OK( + context, + PrecomputeZeroPointTimesWeightWithBias( + context, output_state_zero_point, recurrent_to_cell_weights, nullptr, + &(integer_lstm_params->recurrent_to_cell_effective_bias))); + + // Output gate. + TfLiteTensor* output_gate_bias = is_layer_norm + ? nullptr + : micro_context->AllocateTempInputTensor( + node, kLstmOutputGateBiasTensor); + TF_LITE_ENSURE_OK( + context, + PrecomputeZeroPointTimesWeightWithBias( + context, input_zero_point, input_to_output_weights, output_gate_bias, + &(integer_lstm_params->input_to_output_effective_bias))); + + TF_LITE_ENSURE_OK( + context, + PrecomputeZeroPointTimesWeightWithBias( + context, output_state_zero_point, recurrent_to_output_weights, + nullptr, &(integer_lstm_params->recurrent_to_output_effective_bias))); + + // Input gate. The calculation is only meaningful for non-cifg case. + TfLiteTensor* input_gate_bias = is_layer_norm + ? nullptr + : micro_context->AllocateTempInputTensor( + node, kLstmInputGateBiasTensor); + TF_LITE_ENSURE_OK( + context, + PrecomputeZeroPointTimesWeightWithBias( + context, input_zero_point, input_to_input_weights, input_gate_bias, + &(integer_lstm_params->input_to_input_effective_bias))); + TF_LITE_ENSURE_OK( + context, + PrecomputeZeroPointTimesWeightWithBias( + context, output_state_zero_point, recurrent_to_input_weights, nullptr, + &(integer_lstm_params->recurrent_to_input_effective_bias))); + + // Projection bias. The calculation is only meaningful for with projection. + TF_LITE_ENSURE_OK(context, + PrecomputeZeroPointTimesWeightWithBias( + context, hidden_zp, projection_weights, projection_bias, + &(integer_lstm_params->projection_effective_bias))); + + if (input != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input); + } + if (output_state != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output_state); + } + if (input_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_input_weights); + } + if (input_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_forget_weights); + } + if (input_to_cell_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_cell_weights); + } + if (input_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_output_weights); + } + if (recurrent_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_input_weights); + } + if (recurrent_to_forget_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_forget_weights); + } + if (recurrent_to_cell_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_cell_weights); + } + if (recurrent_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_output_weights); + } + if (projection_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(projection_weights); + } + if (projection_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(projection_bias); + } + if (forget_gate_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(forget_gate_bias); + } + if (cell_gate_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_gate_bias); + } + if (output_gate_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output_gate_bias); + } + if (input_gate_bias != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_gate_bias); + } + + if (intermediate != nullptr) { + micro_context->DeallocateTempTfLiteTensor(intermediate); + } + + return kTfLiteOk; +} + +// Resize the output and state tensors based on the sizes of the input tensors. +// Allocate a temporary scratch tensor. Also check that the sizes of the input +// tensors match each other. +TfLiteStatus UnidirectionalSequenceLstmPrepare(TfLiteContext* context, + TfLiteNode* node) { + UnidirectionalSequenceLstmOpData* op_data = + reinterpret_cast(node->user_data); + + MicroContext* micro_context = GetMicroContext(context); + + // Check we have all the inputs and outputs we need. + bool use_layer_norm = false; + if (node->inputs->size == 24) { + TfLiteTensor* forget_layer_norm_coefficients = + micro_context->AllocateTempInputTensor( + node, kLstmForgetLayerNormCoefficientsTensor); + if (forget_layer_norm_coefficients == nullptr) { + use_layer_norm = false; + } else { + use_layer_norm = true; + } + if (forget_layer_norm_coefficients != nullptr) { + micro_context->DeallocateTempTfLiteTensor(forget_layer_norm_coefficients); + } + } else if (node->inputs->size == 20) { + // This is deprecated and is only kept here for backward compatibility. + use_layer_norm = false; + } else { + MicroPrintf("The LSTM Full kernel expects 20 or 24 inputs. Got %d inputs", + node->inputs->size); + return kTfLiteError; + } + TF_LITE_ENSURE_EQ(context, node->outputs->size, 1); + op_data->use_layer_norm = use_layer_norm; + + // Inferring batch size, number of outputs and sequence length and + // number of cells from the input tensors. + TfLiteTensor* input = + micro_context->AllocateTempInputTensor(node, kLstmInputTensor); + op_data->input_zero_point = input->params.zero_point; + const bool is_integer = input->type == kTfLiteInt8; + TF_LITE_ENSURE(context, input->dims->size > 1); + const auto* params = + reinterpret_cast( + node->builtin_data); + const bool time_major = params->time_major; + const int n_batch = time_major ? input->dims->data[1] : input->dims->data[0]; + const int n_input = input->dims->data[2]; + + TfLiteTensor* input_to_output_weights = + micro_context->AllocateTempInputTensor(node, + kLstmInputToOutputWeightsTensor); + const int n_cell = input_to_output_weights->dims->data[0]; + TF_LITE_ENSURE_EQ(context, input_to_output_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, input_to_output_weights->dims->data[1], n_input); + + TfLiteTensor* recurrent_to_output_weights = + micro_context->AllocateTempInputTensor( + node, kLstmRecurrentToOutputWeightsTensor); + TF_LITE_ENSURE_EQ(context, recurrent_to_output_weights->dims->size, 2); + TF_LITE_ENSURE_EQ(context, recurrent_to_output_weights->dims->data[0], + n_cell); + const int n_output = recurrent_to_output_weights->dims->data[1]; + + // Check that input tensor dimensions matches with each other. + TF_LITE_ENSURE_OK( + context, CheckInputTensorDimensions(context, node, n_input, n_output, + n_cell, use_layer_norm, is_integer)); + + // Get the pointer to output, output_state and cell_state buffer tensors. + TfLiteTensor* output = + micro_context->AllocateTempOutputTensor(node, kLstmOutputTensor); + + TfLiteTensor* output_state = + micro_context->AllocateTempInputTensor(node, kLstmOutputStateTensor); + TF_LITE_ENSURE(context, output_state != nullptr); + TF_LITE_ENSURE(context, output_state->is_variable); + op_data->output_state_zero_point = output_state->params.zero_point; + TfLiteTensor* cell_state = + micro_context->AllocateTempInputTensor(node, kLstmCellStateTensor); + TF_LITE_ENSURE(context, cell_state != nullptr); + TF_LITE_ENSURE(context, cell_state->is_variable); + + // Check the shape of input state tensors. + // These tensor may be 1D or 2D. It's fine as long as the total size is + // correct. + TF_LITE_ENSURE_EQ(context, NumElements(output_state), n_batch * n_output); + TF_LITE_ENSURE_EQ(context, NumElements(cell_state), n_batch * n_cell); + + // Check the shape of output tensor against that of input tensor + TF_LITE_ENSURE_EQ(context, output->dims->size, 3); + TF_LITE_ENSURE_EQ(context, input->dims->data[0], output->dims->data[0]); + TF_LITE_ENSURE_EQ(context, input->dims->data[1], output->dims->data[1]); + TF_LITE_ENSURE_EQ(context, output->dims->data[2], n_output); + + if (is_integer) { + const int num_intermediate_tensors = node->intermediates->size; + TF_LITE_ENSURE(context, num_intermediate_tensors == 5); + } + + TfLiteTensor* input_to_input_weights = micro_context->AllocateTempInputTensor( + node, kLstmInputToInputWeightsTensor); + + const bool use_cifg = (input_to_input_weights == nullptr); + + // Create a primary scratch buffer for hybrid and float + // If is_integer, primary scratch buffer has a different size + if (!is_integer) { + int scratch_buffer_size[2]; + scratch_buffer_size[0] = n_batch; + + if (use_cifg) { + // Reserving space for Cell, Forget, Output gates + scratch_buffer_size[1] = n_cell * 3; + } else { + // Reserving space for Input, Cell, Forget, Output gates + scratch_buffer_size[1] = n_cell * 4; + } + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, + scratch_buffer_size[0] * scratch_buffer_size[1] * + TfLiteTypeGetSize(input->type), + &(op_data->scratch_index[kPrimaryScratchBuffer]))); + } + + if (IsHybridOp(input, input_to_output_weights)) { + TF_LITE_ENSURE(context, kNumHybridTempBuffers <= scratch_index_size); + + TF_LITE_ENSURE_OK(context, SetHybridScales(context, node)); + + op_data->compute_row_sums = true; + + // Allocate temporary tensors to store quantized values of input, + // output_state and cell_state tensors. + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, + GetTensorShape(input).FlatSize() * + TfLiteTypeGetSize(input_to_output_weights->type), + &(op_data->scratch_index[kInputQuantized]))); + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, + GetTensorShape(output_state).FlatSize() * + TfLiteTypeGetSize(input_to_output_weights->type), + &(op_data->scratch_index[kOutputStateQuantized]))); + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, + GetTensorShape(cell_state).FlatSize() * + TfLiteTypeGetSize(input_to_output_weights->type), + &(op_data->scratch_index[kCellStateQuantized]))); + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, n_batch * TfLiteTypeGetSize(kTfLiteFloat32), + &(op_data->scratch_index[kScales]))); + + // Allocate temporary buffers to store scaling factors and product scaling + // factors. The latter is a convenience storage which allows to quantize + // a vector once (which produces the scaling factors) and multiply it with + // different matrices (which requires multiplying the scaling factors with + // the scaling factor of the matrix). + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, n_batch * TfLiteTypeGetSize(kTfLiteFloat32), + &(op_data->scratch_index[kInputScalingFactors]))); + + TF_LITE_ENSURE_OK( + context, context->RequestScratchBufferInArena( + context, n_batch * TfLiteTypeGetSize(kTfLiteFloat32), + &(op_data->scratch_index[kOutputStateScalingFactors]))); + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, n_batch * TfLiteTypeGetSize(kTfLiteFloat32), + &(op_data->scratch_index[kProductScalingFactors]))); + + // Allocate a temporary buffer to store the recovered cell weights. Since + // this is used for diagonal matrices, only need to store n_cell values. + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, n_cell * TfLiteTypeGetSize(kTfLiteFloat32), + &(op_data->scratch_index[kRecoveredCellWeights]))); + + // Allocate a temporary buffer to store the accumulated int32 values. + TF_LITE_ENSURE_OK( + context, + context->RequestScratchBufferInArena( + context, n_cell * n_batch * TfLiteTypeGetSize(kTfLiteInt32), + &(op_data->scratch_index[kAccumScratch]))); + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, n_batch * TfLiteTypeGetSize(kTfLiteFloat32), + &(op_data->scratch_index[kInputZeroPoints]))); + + TF_LITE_ENSURE_OK(context, + context->RequestScratchBufferInArena( + context, n_batch * TfLiteTypeGetSize(kTfLiteFloat32), + &(op_data->scratch_index[kOutputStateZeroPoints]))); + + int row_sums_rows = use_cifg ? 6 : 8; + TfLiteTensor* projection_weights = micro_context->AllocateTempInputTensor( + node, kLstmProjectionWeightsTensor); + if (projection_weights != nullptr) { + row_sums_rows += ceil(static_cast(n_output) / n_cell); + } + op_data->row_sums_size = row_sums_rows; + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + op_data->row_sums = static_cast(context->AllocatePersistentBuffer( + context, row_sums_rows * n_cell * sizeof(int32_t))); + if (projection_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(projection_weights); + } + } + + if (is_integer) { + // Integer UnidirectionalSequenceLSTM prepare function for 8x8->16. + // This code path needs 5 intermediate tensors per Op. + // Populate quantization parameters. + PopulateQuantizedLstmParams8x8_16(context, node, + &op_data->integer_lstm_param); + // Allocate scratch buffer. Need 4 16-bit buffer with size n_batch * n_cell + // and 1 8-bit buffer with size n_batch * n_cell. For integer + // UnidirectionalSequenceLSTM, we do not need the extra 32-bit buffer. + for (int i = 0; i < 5; ++i) { + TfLiteType buffer_type = kTfLiteInt16; + + if (i == 4) { + buffer_type = kTfLiteInt8; + } + + TF_LITE_ENSURE_OK( + context, + context->RequestScratchBufferInArena( + context, n_batch * n_cell * TfLiteTypeGetSize(buffer_type), + &(op_data->scratch_index[i]))); + } + + // Populate precomputed zp * weight. + TF_LITE_ENSURE_OK(context, PopulatePrecomputedZPTimesWeightsWithBias( + context, op_data, node)); + } + + if (input != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input); + } + if (input_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_output_weights); + } + if (recurrent_to_output_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(recurrent_to_output_weights); + } + if (output != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output); + } + if (output_state != nullptr) { + micro_context->DeallocateTempTfLiteTensor(output_state); + } + if (cell_state != nullptr) { + micro_context->DeallocateTempTfLiteTensor(cell_state); + } + + if (input_to_input_weights != nullptr) { + micro_context->DeallocateTempTfLiteTensor(input_to_input_weights); + } + return kTfLiteOk; +} + +TfLiteStatus UnidirectionalSequenceLstmEval(TfLiteContext* context, + TfLiteNode* node) { + TFLITE_DCHECK(context->GetScratchBuffer != nullptr); + + const auto* params = + reinterpret_cast( + node->builtin_data); + const UnidirectionalSequenceLstmOpData* op_data = + reinterpret_cast(node->user_data); + const bool use_layer_norm = op_data->use_layer_norm; + const bool time_major = params->time_major; + + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kLstmInputTensor); + + const TfLiteEvalTensor* input_to_input_weights = tflite::micro::GetEvalInput( + context, node, kLstmInputToInputWeightsTensor); + + const TfLiteEvalTensor* input_to_forget_weights = tflite::micro::GetEvalInput( + context, node, kLstmInputToForgetWeightsTensor); + + const TfLiteEvalTensor* input_to_cell_weights = + tflite::micro::GetEvalInput(context, node, kLstmInputToCellWeightsTensor); + + const TfLiteEvalTensor* input_to_output_weights = tflite::micro::GetEvalInput( + context, node, kLstmInputToOutputWeightsTensor); + + const TfLiteEvalTensor* recurrent_to_input_weights = + tflite::micro::GetEvalInput(context, node, + kLstmRecurrentToInputWeightsTensor); + + const TfLiteEvalTensor* recurrent_to_forget_weights = + tflite::micro::GetEvalInput(context, node, + kLstmRecurrentToForgetWeightsTensor); + + const TfLiteEvalTensor* recurrent_to_cell_weights = + tflite::micro::GetEvalInput(context, node, + kLstmRecurrentToCellWeightsTensor); + + const TfLiteEvalTensor* recurrent_to_output_weights = + tflite::micro::GetEvalInput(context, node, + kLstmRecurrentToOutputWeightsTensor); + + const TfLiteEvalTensor* cell_to_input_weights = + tflite::micro::GetEvalInput(context, node, kLstmCellToInputWeightsTensor); + + const TfLiteEvalTensor* cell_to_forget_weights = tflite::micro::GetEvalInput( + context, node, kLstmCellToForgetWeightsTensor); + + const TfLiteEvalTensor* cell_to_output_weights = tflite::micro::GetEvalInput( + context, node, kLstmCellToOutputWeightsTensor); + + const TfLiteEvalTensor* input_gate_bias = + tflite::micro::GetEvalInput(context, node, kLstmInputGateBiasTensor); + + const TfLiteEvalTensor* forget_gate_bias = + tflite::micro::GetEvalInput(context, node, kLstmForgetGateBiasTensor); + + const TfLiteEvalTensor* cell_gate_bias = + tflite::micro::GetEvalInput(context, node, kLstmCellGateBiasTensor); + + const TfLiteEvalTensor* output_gate_bias = + tflite::micro::GetEvalInput(context, node, kLstmOutputGateBiasTensor); + + const TfLiteEvalTensor* projection_weights = + tflite::micro::GetEvalInput(context, node, kLstmProjectionWeightsTensor); + + const TfLiteEvalTensor* projection_bias = + tflite::micro::GetEvalInput(context, node, kLstmProjectionBiasTensor); + + TfLiteEvalTensor* output_state = + tflite::micro::GetMutableEvalInput(context, node, kLstmOutputStateTensor); + + TfLiteEvalTensor* cell_state = + tflite::micro::GetMutableEvalInput(context, node, kLstmCellStateTensor); + + TFLITE_DCHECK(cell_state != nullptr); + + const TfLiteEvalTensor* input_layer_norm_coefficients = + use_layer_norm ? tflite::micro::GetEvalInput( + context, node, kLstmInputLayerNormCoefficientsTensor) + : nullptr; + const TfLiteEvalTensor* forget_layer_norm_coefficients = + use_layer_norm + ? tflite::micro::GetEvalInput(context, node, + kLstmForgetLayerNormCoefficientsTensor) + : nullptr; + const TfLiteEvalTensor* cell_layer_norm_coefficients = + use_layer_norm ? tflite::micro::GetEvalInput( + context, node, kLstmCellLayerNormCoefficientsTensor) + : nullptr; + const TfLiteEvalTensor* output_layer_norm_coefficients = + use_layer_norm + ? tflite::micro::GetEvalInput(context, node, + kLstmOutputLayerNormCoefficientsTensor) + : nullptr; + + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kLstmOutputTensor); + + // Copy out the LSTM specific params so they can be passed in the function. + TfLiteLSTMParams lstm_params; + lstm_params.activation = params->activation; + lstm_params.cell_clip = params->cell_clip; + lstm_params.proj_clip = params->proj_clip; + lstm_params.asymmetric_quantize_inputs = params->asymmetric_quantize_inputs; + + switch (input_to_output_weights->type) { + case kTfLiteFloat32: { + // Index the scratch buffers pointers to the global scratch buffer. + return EvalFloatLstm( + input, input_to_input_weights, input_to_forget_weights, + input_to_cell_weights, input_to_output_weights, + recurrent_to_input_weights, recurrent_to_forget_weights, + recurrent_to_cell_weights, recurrent_to_output_weights, + cell_to_input_weights, cell_to_forget_weights, cell_to_output_weights, + input_layer_norm_coefficients, forget_layer_norm_coefficients, + cell_layer_norm_coefficients, output_layer_norm_coefficients, + /*aux_input=*/nullptr, + /*aux_input_to_input_weights=*/nullptr, + /*aux_input_to_forget_weights=*/nullptr, + /*aux_input_to_cell_weights=*/nullptr, + /*aux_input_to_output_weights=*/nullptr, input_gate_bias, + forget_gate_bias, cell_gate_bias, output_gate_bias, + projection_weights, projection_bias, &lstm_params, + /*forward_sequence=*/true, time_major, + /*output_offset=*/0, + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kPrimaryScratchBuffer])), + output_state, cell_state, output); + } break; + case kTfLiteUInt8: + case kTfLiteInt8: { + const bool is_hybrid = input->type == kTfLiteFloat32; + if (is_hybrid) { + // Index the scratch buffers pointers to the global scratch buffer. + UnidirectionalSequenceLstmOpData* op_data_rw = + reinterpret_cast( + node->user_data); + return EvalHybridLstm( + &(op_data->hybrid_lstm_scales), input, input_to_input_weights, + /*input_to_input_weights_ledger*/ nullptr, input_to_forget_weights, + /*input_to_forget_weights_ledger*/ nullptr, input_to_cell_weights, + /*input_to_cell_weights_ledger*/ nullptr, input_to_output_weights, + /*input_to_output_weights_ledger*/ nullptr, + recurrent_to_input_weights, + /*recurrent_to_input_weights_ledger*/ nullptr, + recurrent_to_forget_weights, + /*recurrent_to_forget_weights_ledger*/ nullptr, + recurrent_to_cell_weights, + /*recurrent_to_cell_weights_ledger*/ nullptr, + recurrent_to_output_weights, + /*recurrent_to_output_weights_ledger*/ nullptr, + cell_to_input_weights, cell_to_forget_weights, + cell_to_output_weights, input_layer_norm_coefficients, + forget_layer_norm_coefficients, cell_layer_norm_coefficients, + output_layer_norm_coefficients, + /*aux_input=*/nullptr, + /*aux_input_to_input_weights=*/nullptr, + /*aux_input_to_forget_weights=*/nullptr, + /*aux_input_to_cell_weights=*/nullptr, + /*aux_input_to_output_weights=*/nullptr, input_gate_bias, + forget_gate_bias, cell_gate_bias, output_gate_bias, + projection_weights, /*projection_weights_ledger*/ nullptr, + projection_bias, &lstm_params, + /*forward_sequence=*/true, time_major, + /*output_offset=*/0, + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kPrimaryScratchBuffer])), + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kInputScalingFactors])), + /*aux_input_sf=*/nullptr, + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kOutputStateScalingFactors])), + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kProductScalingFactors])), + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kRecoveredCellWeights])), + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kInputQuantized])), + /*aux_input_quantized=*/nullptr, + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kOutputStateQuantized])), + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kCellStateQuantized])), + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kScales])), + output_state, cell_state, + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kAccumScratch])), + output, + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kInputZeroPoints])), + /*aux_input_zp=*/nullptr, + reinterpret_cast(context->GetScratchBuffer( + context, op_data->scratch_index[kOutputStateZeroPoints])), + op_data_rw->row_sums, op_data_rw->row_sums_size, + &op_data_rw->compute_row_sums); + } else { + return EvalInteger8x8_16Lstm( + input, input_to_input_weights, input_to_forget_weights, + input_to_cell_weights, input_to_output_weights, + recurrent_to_input_weights, recurrent_to_forget_weights, + recurrent_to_cell_weights, recurrent_to_output_weights, + cell_to_input_weights, cell_to_forget_weights, + cell_to_output_weights, input_layer_norm_coefficients, + forget_layer_norm_coefficients, cell_layer_norm_coefficients, + output_layer_norm_coefficients, input_gate_bias, forget_gate_bias, + cell_gate_bias, output_gate_bias, projection_weights, + projection_bias, &lstm_params, /*forward_sequence=*/true, + time_major, &op_data->integer_lstm_param, + op_data->output_state_zero_point, output_state, cell_state, output, + reinterpret_cast( + context->GetScratchBuffer(context, op_data->scratch_index[0])), + reinterpret_cast( + context->GetScratchBuffer(context, op_data->scratch_index[1])), + reinterpret_cast( + context->GetScratchBuffer(context, op_data->scratch_index[2])), + reinterpret_cast( + context->GetScratchBuffer(context, op_data->scratch_index[3])), + reinterpret_cast( + context->GetScratchBuffer(context, op_data->scratch_index[4])), + nullptr); + } + } break; + default: + MicroPrintf("Type %s is not currently supported.", + TfLiteTypeGetName(input_to_output_weights->type)); + return kTfLiteError; + } +} + +} // namespace + +TfLiteRegistration Register_UNIDIRECTIONAL_SEQUENCE_LSTM() { + return tflite::micro::RegisterOp(UnidirectionalSequenceLstmInit, + UnidirectionalSequenceLstmPrepare, + UnidirectionalSequenceLstmEval); +} + +} // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/unidirectional_sequence_lstm_test_config.h b/code/components/tflite-lib/tensorflow/lite/micro/kernels/unidirectional_sequence_lstm_test_config.h new file mode 100644 index 00000000..e37c0efd --- /dev/null +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/unidirectional_sequence_lstm_test_config.h @@ -0,0 +1,244 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_UNIDIRECTIONAL_SEQUENCE_LSTM_TEST_CONFIG_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_UNIDIRECTIONAL_SEQUENCE_LSTM_TEST_CONFIG_H_ + +#include "tensorflow/lite/c/common.h" + +namespace tflite { +namespace testing { + +// TODO(b/230666079) enable below tests for xtensa when the xtensa +// kernel is reconciled with reference kernel +#if !defined(XTENSA) + +typedef struct LstmIntegerTestConfig { + const int n_batch; + const int n_input; + const int n_cell; + const int n_output; + const int sequence_length; + const bool time_major; + const bool use_cifg; + const bool use_peephole; + const bool use_projection_weights; + const bool use_projection_bias; + const bool use_layer_norm; + const bool use_8x8_8_implementation; + float intermediate_scale[5][2]; + int intermediate_zp[5][2]; + TfLiteAffineQuantization* intermediate_qparam; + + const float* input; + int8_t* input_quant; + + const float* input_to_input_weights; + int8_t* lstm_i2i_quant; + const float* input_to_forget_weights; + int8_t* lstm_i2f_quant; + const float* input_to_cell_weights; + int8_t* lstm_i2c_quant; + const float* input_to_output_weights; + int8_t* lstm_i2o_quant; + + const float* recurrent_to_input_weights; + int8_t* lstm_r2i_quant; + const float* recurrent_to_forget_weights; + int8_t* lstm_r2f_quant; + const float* recurrent_to_cell_weights; + int8_t* lstm_r2c_quant; + const float* recurrent_to_output_weights; + int8_t* lstm_r2o_quant; + + const float* cell_to_input_weights; + int16_t* lstm_c2i_quant; + const float* cell_to_forget_weights; + int16_t* lstm_c2f_quant; + const float* cell_to_output_weights; + int16_t* lstm_c2o_quant; + + const float* input_gate_bias; + int32_t* lstm_igate_bias_quant; + const float* forget_gate_bias; + int32_t* lstm_fgate_bias_quant; + const float* cell_gate_bias; + int32_t* lstm_cgate_bias_quant; + const float* output_gate_bias; + int32_t* lstm_ogate_bias_quant; + + const float* projection_weights; + int8_t* lstm_proj_w_quant; + const float* projection_bias; + int32_t* projection_bias_quant; + + int16_t* output_state; + int16_t* cell_state; + + const float* input_layer_norm_coefficients; + int16_t* lstm_input_layer_norm_coeff_quant; + const float* forget_layer_norm_coefficients; + int16_t* lstm_forget_layer_norm_coeff_quant; + const float* cell_layer_norm_coefficients; + int16_t* lstm_cell_layer_norm_coeff_quant; + const float* output_layer_norm_coefficients; + int16_t* lstm_output_layer_norm_coeff_quant; + + int8_t* output; + const int8_t* expected_output; + + bool asymmetric_quantize_inputs; + const float ranges[25][2]; +} LstmIntegerTestConfig; + +typedef struct LstmFloatTestConfig { + const int n_batch; + const int n_input; + const int n_cell; + const int n_output; + const int sequence_length; + const bool time_major; + const bool use_cifg; + const bool use_peephole; + const bool use_projection_weights; + const bool use_projection_bias; + const bool use_layer_norm; + const float cell_clip; + const float proj_clip; + + const float* input_original; + float* input; + + const float* input_to_input_weights; + const float* input_to_forget_weights; + const float* input_to_cell_weights; + const float* input_to_output_weights; + + const float* recurrent_to_input_weights; + const float* recurrent_to_forget_weights; + const float* recurrent_to_cell_weights; + const float* recurrent_to_output_weights; + + const float* cell_to_input_weights; + const float* cell_to_forget_weights; + const float* cell_to_output_weights; + + const float* input_gate_bias; + const float* forget_gate_bias; + const float* cell_gate_bias; + const float* output_gate_bias; + + const float* projection_weights; + const float* projection_bias; + + float* output_state; + float* cell_state; + + const float* input_layer_norm_coefficients; + const float* forget_layer_norm_coefficients; + const float* cell_layer_norm_coefficients; + const float* output_layer_norm_coefficients; + + float* output; + const float* expected_output_original; + float* expected_output; +} LstmFloatTestConfig; + +typedef struct LstmWeightQuantizationBuffers { + int8_t* lstm_i2i_quant; + float* lstm_i2i_scale; + int* lstm_i2i_zp; + TfLiteAffineQuantization* lstm_i2i_qparam; + + int8_t* lstm_i2f_quant; + float* lstm_i2f_scale; + int* lstm_i2f_zp; + TfLiteAffineQuantization* lstm_i2f_qparam; + + int8_t* lstm_i2c_quant; + float* lstm_i2c_scale; + int* lstm_i2c_zp; + TfLiteAffineQuantization* lstm_i2c_qparam; + + int8_t* lstm_i2o_quant; + float* lstm_i2o_scale; + int* lstm_i2o_zp; + TfLiteAffineQuantization* lstm_i2o_qparam; + + int8_t* lstm_r2i_quant; + float* lstm_r2i_scale; + int* lstm_r2i_zp; + TfLiteAffineQuantization* lstm_r2i_qparam; + + int8_t* lstm_r2f_quant; + float* lstm_r2f_scale; + int* lstm_r2f_zp; + TfLiteAffineQuantization* lstm_r2f_qparam; + + int8_t* lstm_r2c_quant; + float* lstm_r2c_scale; + int* lstm_r2c_zp; + TfLiteAffineQuantization* lstm_r2c_qparam; + + int8_t* lstm_r2o_quant; + float* lstm_r2o_scale; + int* lstm_r2o_zp; + TfLiteAffineQuantization* lstm_r2o_qparam; + + int8_t* lstm_c2i_quant; + float* lstm_c2i_scale; + int* lstm_c2i_zp; + TfLiteAffineQuantization* lstm_c2i_qparam; + + int8_t* lstm_c2f_quant; + float* lstm_c2f_scale; + int* lstm_c2f_zp; + TfLiteAffineQuantization* lstm_c2f_qparam; + + int8_t* lstm_c2o_quant; + float* lstm_c2o_scale; + int* lstm_c2o_zp; + TfLiteAffineQuantization* lstm_c2o_qparam; + + int8_t* lstm_proj_w_quant; + float* lstm_proj_w_scale; + int* lstm_proj_w_zp; + TfLiteAffineQuantization* lstm_proj_w_qparam; +} LstmWeightQuantizationBuffers; + +extern LstmIntegerTestConfig lstm_integer_no_peephole_config; + +extern LstmIntegerTestConfig lstm_integer_peephole_config; + +extern LstmFloatTestConfig lstm_no_cifg_no_peephole_no_proj_config; + +extern LstmFloatTestConfig lstm_cifg_peephole_no_proj_config; + +extern LstmFloatTestConfig lstm_no_cifg_peephole_proj_config; + +extern LstmFloatTestConfig lstm_no_cifg_peephole_proj_bias_config; + +extern LstmWeightQuantizationBuffers lstm_no_cifg_no_peephole_no_proj_buffers; + +extern LstmWeightQuantizationBuffers lstm_cifg_peephole_no_proj_buffers; + +extern LstmWeightQuantizationBuffers lstm_no_cifg_peephole_proj_buffers; + +extern LstmFloatTestConfig cifg_peephole_no_proj_config_layer_norm; + +#endif // !defined(XTENSA) +} // namespace testing +} // namespace tflite + +#endif // TENSORFLOW_LITE_MICRO_KERNELS_UNIDIRECTIONAL_SEQUENCE_LSTM_TEST_CONFIG_H_ diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/unpack.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/unpack.cc index 13bb7dcf..d199add0 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/unpack.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/unpack.cc @@ -103,14 +103,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace unpack TfLiteRegistration Register_UNPACK() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/nullptr, - /*invoke=*/unpack::Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, nullptr, unpack::Eval); } } // namespace micro diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/var_handle.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/var_handle.cc index 8354c918..db044f3f 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/var_handle.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/var_handle.cc @@ -87,14 +87,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace. TfLiteRegistration Register_VAR_HANDLE() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/while.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/while.cc index 576c19b0..811c9eae 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/while.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/while.cc @@ -127,14 +127,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace. TfLiteRegistration Register_WHILE() { - return {/*init=*/Init, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(Init, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/kernels/zeros_like.cc b/code/components/tflite-lib/tensorflow/lite/micro/kernels/zeros_like.cc index 733564c9..fd6e6612 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/kernels/zeros_like.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/kernels/zeros_like.cc @@ -81,14 +81,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace TfLiteRegistration Register_ZEROS_LIKE() { - return {/*init=*/nullptr, - /*free=*/nullptr, - /*prepare=*/Prepare, - /*invoke=*/Eval, - /*profiling_string=*/nullptr, - /*builtin_code=*/0, - /*custom_name=*/nullptr, - /*version=*/0}; + return tflite::micro::RegisterOp(nullptr, Prepare, Eval); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_allocation_info.cc b/code/components/tflite-lib/tensorflow/lite/micro/micro_allocation_info.cc index ab313e66..edab2b83 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_allocation_info.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_allocation_info.cc @@ -16,8 +16,10 @@ limitations under the License. #include "tensorflow/lite/c/c_api_types.h" #include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/micro/memory_helpers.h" #include "tensorflow/lite/micro/memory_planner/greedy_memory_planner.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" namespace tflite { @@ -148,6 +150,30 @@ TfLiteStatus AllocationInfoBuilder::FreeAllocationInfo() { return kTfLiteOk; } +TfLiteStatus AllocationInfoBuilder::ValidateSubgraph( + const SubGraph* subgraph, TfLiteEvalTensor* eval_tensors) { + uint32_t operators_size = NumSubgraphOperators(subgraph); + + for (uint32_t i = 0; i < operators_size; i++) { + const auto op = subgraph->operators()->Get(i); + for (size_t n = 0; + op->intermediates() != nullptr && n < op->intermediates()->size(); + n++) { + const int tensor_index = op->intermediates()->Get(n); + size_t tensor_size = -1; + TF_LITE_ENSURE_STATUS(TfLiteEvalTensorByteLength( + &eval_tensors[tensor_index], &tensor_size)); + if (tensor_size != 0) { + MicroPrintf( + "Does not support intermediate tensor with non-zero size: %d", + tensor_size); + return kTfLiteError; + } + } + } + return kTfLiteOk; +} + TfLiteStatus AllocationInfoBuilder::InitializeAllocationInfo( const int32_t* offline_offsets, SubgraphAllocations* allocations) { AllocationInfo* allocation_info = info_.allocation_info; @@ -158,6 +184,10 @@ TfLiteStatus AllocationInfoBuilder::InitializeAllocationInfo( TfLiteEvalTensor* eval_tensors = allocations[subgraph_idx].tensors; AllocationInfo* subgraph_allocation_info = &allocation_info[info_.subgraph_offsets[subgraph_idx]]; + + // Ensure constraints are met. + TF_LITE_ENSURE_STATUS(ValidateSubgraph(subgraph, eval_tensors)); + for (size_t i = 0; i < subgraph->tensors()->size(); ++i) { AllocationInfo* current = &subgraph_allocation_info[i]; current->output_ptr = &(eval_tensors[i].data.data); @@ -167,8 +197,10 @@ TfLiteStatus AllocationInfoBuilder::InitializeAllocationInfo( current->first_created = kUninitializedLifetime; current->last_used = kUninitializedLifetime; - current->needs_allocating = (eval_tensors[i].data.data == nullptr) && - (!subgraph->tensors()->Get(i)->is_variable()); + current->needs_allocating = + (eval_tensors[i].data.data == nullptr) && + (!subgraph->tensors()->Get(i)->is_variable()) && + (current->bytes != 0); if (offline_offsets) { current->offline_offset = offline_offsets[i]; } else { @@ -181,8 +213,8 @@ TfLiteStatus AllocationInfoBuilder::InitializeAllocationInfo( &allocation_info[info_.scratch_offset]; for (size_t i = 0; i < info_.scratch_buffer_count; i++) { AllocationInfo* current = &scratch_allocation_info[i]; - current->first_created = -1; - current->last_used = -1; + current->first_created = kUninitializedLifetime; + current->last_used = kUninitializedLifetime; current->needs_allocating = true; current->offline_offset = kOnlinePlannedBuffer; } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_allocation_info.h b/code/components/tflite-lib/tensorflow/lite/micro/micro_allocation_info.h index af303307..4ea435b3 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_allocation_info.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_allocation_info.h @@ -67,8 +67,13 @@ class AllocationInfoBuilder { INonPersistentBufferAllocator* non_persistent_allocator, ErrorReporter* reporter) : model_(model), - non_persistent_allocator_(non_persistent_allocator), - reporter_(reporter) {} + non_persistent_allocator_(non_persistent_allocator) +#if !defined(TF_LITE_STRIP_ERROR_STRINGS) + , + reporter_(reporter) +#endif + { + } // Check if model contains offline planned buffer offsets. // - If there's no metadata available, offline_planner_offsets is not set @@ -128,9 +133,15 @@ class AllocationInfoBuilder { // count monotonically increases through the lifetime marking process. void UpdateLastUsed(AllocationInfo* current, int allocation_scope_count); + // Validate if a subgraph satisfies assumptions. + TfLiteStatus ValidateSubgraph(const SubGraph* subgraph, + TfLiteEvalTensor* eval_tensors); + const tflite::Model* model_ = nullptr; INonPersistentBufferAllocator* non_persistent_allocator_ = nullptr; +#if !defined(TF_LITE_STRIP_ERROR_STRINGS) ErrorReporter* reporter_ = nullptr; +#endif GraphAllocationInfo info_; int allocation_scope_count_ = 0; diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_allocator.cc b/code/components/tflite-lib/tensorflow/lite/micro/micro_allocator.cc index b71c7502..6b78fdc0 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_allocator.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_allocator.cc @@ -26,6 +26,9 @@ limitations under the License. #include "tensorflow/lite/core/api/op_resolver.h" #include "tensorflow/lite/core/api/tensor_utils.h" #include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.h" +#include "tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.h" +#include "tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h" #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/flatbuffer_utils.h" #include "tensorflow/lite/micro/memory_helpers.h" @@ -34,7 +37,6 @@ limitations under the License. #include "tensorflow/lite/micro/micro_allocation_info.h" #include "tensorflow/lite/micro/micro_arena_constants.h" #include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/simple_memory_allocator.h" #include "tensorflow/lite/schema/schema_generated.h" #include "tensorflow/lite/schema/schema_utils.h" @@ -116,6 +118,48 @@ TfLiteStatus CommitPlan(ErrorReporter* error_reporter, return kTfLiteOk; } +IPersistentBufferAllocator* CreatePersistentArenaAllocator(uint8_t* buffer_head, + size_t buffer_size) { + // Align the actually used area by the tail because persistent buffer grows + // from the bottom to top. + uint8_t* aligned_buffer_tail = + AlignPointerDown(buffer_head + buffer_size, MicroArenaBufferAlignment()); + size_t aligned_buffer_size = aligned_buffer_tail - buffer_head; + PersistentArenaBufferAllocator tmp = + PersistentArenaBufferAllocator(buffer_head, aligned_buffer_size); + + // Allocate enough bytes from the buffer to create a + // SingleArenaBufferAllocator. The new instance will use the current adjusted + // tail buffer from the tmp allocator instance. + uint8_t* allocator_buffer = + tmp.AllocatePersistentBuffer(sizeof(PersistentArenaBufferAllocator), + alignof(PersistentArenaBufferAllocator)); + // Use the default copy constructor to populate internal states. + return new (allocator_buffer) PersistentArenaBufferAllocator(tmp); +} + +// NonPersistentBufferAllocator instance is created in the persistent buffer +// because it has to be persistent to keep track of the non-persistent buffer +// information. +INonPersistentBufferAllocator* CreateNonPersistentArenaAllocator( + uint8_t* buffer_head, size_t buffer_size, + IPersistentBufferAllocator* persistent_buffer_allocator) { + uint8_t* allocator_buffer = + persistent_buffer_allocator->AllocatePersistentBuffer( + sizeof(NonPersistentArenaBufferAllocator), + alignof(NonPersistentArenaBufferAllocator)); + // Align the actually used area by the head because persistent buffer grows + // from the head to bottom. + uint8_t* aligned_buffer_head = + AlignPointerUp(buffer_head, MicroArenaBufferAlignment()); + size_t aligned_buffer_size = buffer_head + buffer_size - aligned_buffer_head; + + INonPersistentBufferAllocator* non_persistent_buffer_allocator = + new (allocator_buffer) NonPersistentArenaBufferAllocator( + aligned_buffer_head, aligned_buffer_size); + return non_persistent_buffer_allocator; +} + } // namespace namespace internal { @@ -302,8 +346,8 @@ size_t MicroAllocator::GetDefaultTailUsage(bool is_memory_planner_given) { // TODO(b/208703041): a template version of AlignSizeUp to make expression // shorter. size_t total_size = - AlignSizeUp(sizeof(SimpleMemoryAllocator), - alignof(SimpleMemoryAllocator)) + + AlignSizeUp(sizeof(SingleArenaBufferAllocator), + alignof(SingleArenaBufferAllocator)) + AlignSizeUp(sizeof(MicroAllocator), alignof(MicroAllocator)) + AlignSizeUp(sizeof(MicroBuiltinDataAllocator), alignof(MicroBuiltinDataAllocator)) + @@ -315,7 +359,7 @@ size_t MicroAllocator::GetDefaultTailUsage(bool is_memory_planner_given) { return total_size; } -MicroAllocator::MicroAllocator(SimpleMemoryAllocator* memory_allocator, +MicroAllocator::MicroAllocator(SingleArenaBufferAllocator* memory_allocator, MicroMemoryPlanner* memory_planner, ErrorReporter* error_reporter) : non_persistent_buffer_allocator_(memory_allocator), @@ -324,6 +368,16 @@ MicroAllocator::MicroAllocator(SimpleMemoryAllocator* memory_allocator, error_reporter_(error_reporter), model_is_allocating_(false) {} +MicroAllocator::MicroAllocator( + IPersistentBufferAllocator* persistent_buffer_allocator, + INonPersistentBufferAllocator* non_persistent_buffer_allocator, + MicroMemoryPlanner* memory_planner, ErrorReporter* error_reporter) + : non_persistent_buffer_allocator_(non_persistent_buffer_allocator), + persistent_buffer_allocator_(persistent_buffer_allocator), + memory_planner_(memory_planner), + error_reporter_(error_reporter), + model_is_allocating_(false) {} + MicroAllocator::~MicroAllocator() {} MicroAllocator* MicroAllocator::Create(uint8_t* tensor_arena, size_t arena_size, @@ -332,8 +386,9 @@ MicroAllocator* MicroAllocator::Create(uint8_t* tensor_arena, size_t arena_size, uint8_t* aligned_arena = AlignPointerUp(tensor_arena, MicroArenaBufferAlignment()); size_t aligned_arena_size = tensor_arena + arena_size - aligned_arena; - SimpleMemoryAllocator* memory_allocator = SimpleMemoryAllocator::Create( - error_reporter, aligned_arena, aligned_arena_size); + SingleArenaBufferAllocator* memory_allocator = + SingleArenaBufferAllocator::Create(error_reporter, aligned_arena, + aligned_arena_size); return Create(memory_allocator, memory_planner, error_reporter); } @@ -343,8 +398,9 @@ MicroAllocator* MicroAllocator::Create(uint8_t* tensor_arena, size_t arena_size, uint8_t* aligned_arena = AlignPointerUp(tensor_arena, MicroArenaBufferAlignment()); size_t aligned_arena_size = tensor_arena + arena_size - aligned_arena; - SimpleMemoryAllocator* memory_allocator = SimpleMemoryAllocator::Create( - error_reporter, aligned_arena, aligned_arena_size); + SingleArenaBufferAllocator* memory_allocator = + SingleArenaBufferAllocator::Create(error_reporter, aligned_arena, + aligned_arena_size); // By default create GreedyMemoryPlanner. // If a different MemoryPlanner is needed, use the other api. @@ -356,17 +412,50 @@ MicroAllocator* MicroAllocator::Create(uint8_t* tensor_arena, size_t arena_size, return Create(memory_allocator, memory_planner, error_reporter); } -MicroAllocator* MicroAllocator::Create(SimpleMemoryAllocator* memory_allocator, - MicroMemoryPlanner* memory_planner, - ErrorReporter* error_reporter) { +MicroAllocator* MicroAllocator::Create( + SingleArenaBufferAllocator* memory_allocator, + MicroMemoryPlanner* memory_planner, ErrorReporter* error_reporter) { TFLITE_DCHECK(memory_allocator != nullptr); TFLITE_DCHECK(error_reporter != nullptr); TFLITE_DCHECK(memory_planner != nullptr); uint8_t* allocator_buffer = memory_allocator->AllocatePersistentBuffer( sizeof(MicroAllocator), alignof(MicroAllocator)); - MicroAllocator* allocator = new (allocator_buffer) - MicroAllocator(memory_allocator, memory_planner, error_reporter); + MicroAllocator* allocator = new (allocator_buffer) MicroAllocator( + memory_allocator, memory_allocator, memory_planner, error_reporter); + return allocator; +} + +MicroAllocator* MicroAllocator::Create(uint8_t* persistent_tensor_arena, + size_t persistent_arena_size, + uint8_t* non_persistent_tensor_arena, + size_t non_persistent_arena_size, + ErrorReporter* error_reporter) { + TFLITE_DCHECK(persistent_tensor_arena != nullptr); + TFLITE_DCHECK(non_persistent_tensor_arena != nullptr); + TFLITE_DCHECK(persistent_tensor_arena != non_persistent_tensor_arena); + TFLITE_DCHECK(error_reporter != nullptr); + + IPersistentBufferAllocator* persistent_buffer_allocator = + CreatePersistentArenaAllocator(persistent_tensor_arena, + persistent_arena_size); + INonPersistentBufferAllocator* non_persistent_buffer_allocator = + CreateNonPersistentArenaAllocator(non_persistent_tensor_arena, + non_persistent_arena_size, + persistent_buffer_allocator); + + uint8_t* memory_planner_buffer = + persistent_buffer_allocator->AllocatePersistentBuffer( + sizeof(GreedyMemoryPlanner), alignof(GreedyMemoryPlanner)); + GreedyMemoryPlanner* memory_planner = + new (memory_planner_buffer) GreedyMemoryPlanner(); + + uint8_t* micro_allocator_buffer = + persistent_buffer_allocator->AllocatePersistentBuffer( + sizeof(MicroAllocator), alignof(MicroAllocator)); + MicroAllocator* allocator = new (micro_allocator_buffer) MicroAllocator( + persistent_buffer_allocator, non_persistent_buffer_allocator, + memory_planner, error_reporter); return allocator; } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_allocator.h b/code/components/tflite-lib/tensorflow/lite/micro/micro_allocator.h index d2967c21..7381489d 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_allocator.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_allocator.h @@ -21,10 +21,10 @@ limitations under the License. #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/core/api/flatbuffer_conversions.h" +#include "tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.h" #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/flatbuffer_utils.h" #include "tensorflow/lite/micro/memory_planner/micro_memory_planner.h" -#include "tensorflow/lite/micro/simple_memory_allocator.h" #include "tensorflow/lite/schema/schema_generated.h" namespace tflite { @@ -96,9 +96,9 @@ typedef struct { // // The MicroAllocator simply plans out additional allocations that are required // to standup a model for inference in TF Micro. This class currently relies on -// an additional allocator - SimpleMemoryAllocator - for all allocations from an -// arena. These allocations are divided into head (non-persistent) and tail -// (persistent) regions: +// an additional allocator - SingleArenaBufferAllocator - for all allocations +// from an arena. These allocations are divided into head (non-persistent) and +// tail (persistent) regions: // // Memory layout to help understand how it works // This information could change in the future version. @@ -129,13 +129,24 @@ class MicroAllocator { MicroMemoryPlanner* memory_planner, ErrorReporter* error_reporter); - // Creates a MicroAllocator instance using the provided SimpleMemoryAllocator - // instance and the MemoryPlanner. This allocator instance will use the - // SimpleMemoryAllocator instance to manage allocations internally. - static MicroAllocator* Create(SimpleMemoryAllocator* memory_allocator, + // Creates a MicroAllocator instance using the provided + // SingleArenaBufferAllocator instance and the MemoryPlanner. This allocator + // instance will use the SingleArenaBufferAllocator instance to manage + // allocations internally. + static MicroAllocator* Create(SingleArenaBufferAllocator* memory_allocator, MicroMemoryPlanner* memory_planner, ErrorReporter* error_reporter); + // Creates a MicroAllocator instance using the provided + // SingleArenaBufferAllocator instance and the MemoryPlanner. This allocator + // instance will use the SingleArenaBufferAllocator instance to manage + // allocations internally. + static MicroAllocator* Create(uint8_t* persistent_tensor_arena, + size_t persistent_arena_size, + uint8_t* non_persistent_tensor_arena, + size_t non_persistent_arena_size, + ErrorReporter* error_reporter); + // Returns the fixed amount of memory overhead of MicroAllocator. static size_t GetDefaultTailUsage(bool is_memory_planner_given); @@ -223,7 +234,11 @@ class MicroAllocator { BuiltinDataAllocator* GetBuiltinDataAllocator(); protected: - MicroAllocator(SimpleMemoryAllocator* memory_allocator, + MicroAllocator(SingleArenaBufferAllocator* memory_allocator, + MicroMemoryPlanner* memory_planner, + ErrorReporter* error_reporter); + MicroAllocator(IPersistentBufferAllocator* persistent_buffer_allocator, + INonPersistentBufferAllocator* non_persistent_buffer_allocator, MicroMemoryPlanner* memory_planner, ErrorReporter* error_reporter); virtual ~MicroAllocator(); diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_context.cc b/code/components/tflite-lib/tensorflow/lite/micro/micro_context.cc index 1526b976..9ec694b8 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_context.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_context.cc @@ -80,6 +80,16 @@ TfLiteTensor* MicroContext::AllocateTempOutputTensor(const TfLiteNode* node, return AllocateTempTfLiteTensor(tensor_index); } +TfLiteTensor* MicroContext::AllocateTempIntermediateTensor( + const TfLiteNode* node, int index) { + const int tensor_index = GetTensorIndex(index, node->intermediates->size, + node->intermediates->data); + if (tensor_index < 0) { + return nullptr; + } + return AllocateTempTfLiteTensor(tensor_index); +} + void MicroContext::DeallocateTempTfLiteTensor(TfLiteTensor* tensor) { return allocator_.DeallocateTempTfLiteTensor(tensor); } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_context.h b/code/components/tflite-lib/tensorflow/lite/micro/micro_context.h index 1db2575e..e7be6544 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_context.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_context.h @@ -73,6 +73,13 @@ class MicroContext { virtual TfLiteTensor* AllocateTempOutputTensor(const TfLiteNode* node, int index); + // Returns a temporary TfLiteTensor struct for the specified intermediate + // tensor of a given mode. This is the recommended API over the deprecated + // GetIntermediates/GetIntermediatesSafe to get a temp intermediate tensor. + // The returned tensor shall be freed via calling DeallocateTempTfLiteTensor. + virtual TfLiteTensor* AllocateTempIntermediateTensor(const TfLiteNode* node, + int index); + // Deallocates a temp TfLiteTensor. // Virtual so that it can be faked for kernel tests. virtual void DeallocateTempTfLiteTensor(TfLiteTensor* tensor); diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_mutable_op_resolver.h b/code/components/tflite-lib/tensorflow/lite/micro/micro_mutable_op_resolver.h index 8676189d..4b273417 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_mutable_op_resolver.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_mutable_op_resolver.h @@ -1,4 +1,4 @@ -/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,10 +25,13 @@ limitations under the License. #include "tensorflow/lite/kernels/op_macros.h" #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/kernels/conv.h" +#include "tensorflow/lite/micro/kernels/depthwise_conv.h" #include "tensorflow/lite/micro/kernels/ethosu.h" #include "tensorflow/lite/micro/kernels/fully_connected.h" #include "tensorflow/lite/micro/kernels/micro_ops.h" +#include "tensorflow/lite/micro/kernels/reduce.h" #include "tensorflow/lite/micro/kernels/softmax.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" #include "tensorflow/lite/micro/micro_op_resolver.h" #include "tensorflow/lite/schema/schema_generated.h" @@ -83,22 +86,16 @@ class MicroMutableOpResolver : public MicroOpResolver { // kTfLiteError. TfLiteStatus AddCustom(const char* name, TfLiteRegistration* registration) { if (registrations_len_ >= tOpCount) { - if (error_reporter_) { - TF_LITE_REPORT_ERROR( - error_reporter_, - "Couldn't register custom op '%s', resolver size is too small (%d)", - name, tOpCount); - } + MicroPrintf( + "Couldn't register custom op '%s', resolver size is too" + "small (%d)", + name, tOpCount); return kTfLiteError; } if (FindOp(name) != nullptr) { - if (error_reporter_ != nullptr) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Calling AddCustom for the same op more than once " - "is not supported (Op: %s).", - name); - } + MicroPrintf("Calling AddCustom for the same op more than once "); + MicroPrintf("is not supported (Op: %s).", name); return kTfLiteError; } @@ -119,8 +116,8 @@ class MicroMutableOpResolver : public MicroOpResolver { ParseAbs); } - TfLiteStatus AddAdd() { - return AddBuiltin(BuiltinOperator_ADD, tflite::Register_ADD(), ParseAdd); + TfLiteStatus AddAdd(const TfLiteRegistration& registration = Register_ADD()) { + return AddBuiltin(BuiltinOperator_ADD, registration, ParseAdd); } TfLiteStatus AddAddN() { @@ -207,9 +204,10 @@ class MicroMutableOpResolver : public MicroOpResolver { tflite::Register_DEPTH_TO_SPACE(), ParseDepthToSpace); } - TfLiteStatus AddDepthwiseConv2D() { - return AddBuiltin(BuiltinOperator_DEPTHWISE_CONV_2D, - Register_DEPTHWISE_CONV_2D(), ParseDepthwiseConv2D); + TfLiteStatus AddDepthwiseConv2D( + const TfLiteRegistration& registration = Register_DEPTHWISE_CONV_2D()) { + return AddBuiltin(BuiltinOperator_DEPTHWISE_CONV_2D, registration, + ParseDepthwiseConv2D); } TfLiteStatus AddDequantize() { @@ -222,6 +220,10 @@ class MicroMutableOpResolver : public MicroOpResolver { tflite::Register_DETECTION_POSTPROCESS()); } + TfLiteStatus AddDiv() { + return AddBuiltin(BuiltinOperator_DIV, tflite::Register_DIV(), ParseDiv); + } + TfLiteStatus AddElu() { return AddBuiltin(BuiltinOperator_ELU, tflite::Register_ELU(), ParseElu); } @@ -372,8 +374,7 @@ class MicroMutableOpResolver : public MicroOpResolver { } TfLiteStatus AddMean() { - return AddBuiltin(BuiltinOperator_MEAN, tflite::ops::micro::Register_MEAN(), - ParseReducer); + return AddBuiltin(BuiltinOperator_MEAN, Register_MEAN(), ParseReducer); } TfLiteStatus AddMinimum() { @@ -426,8 +427,8 @@ class MicroMutableOpResolver : public MicroOpResolver { } TfLiteStatus AddReduceMax() { - return AddBuiltin(BuiltinOperator_REDUCE_MAX, - tflite::ops::micro::Register_REDUCE_MAX(), ParseReducer); + return AddBuiltin(BuiltinOperator_REDUCE_MAX, Register_REDUCE_MAX(), + ParseReducer); } TfLiteStatus AddRelu() { @@ -528,6 +529,10 @@ class MicroMutableOpResolver : public MicroOpResolver { return AddBuiltin(BuiltinOperator_SUB, tflite::Register_SUB(), ParseSub); } + TfLiteStatus AddSum() { + return AddBuiltin(BuiltinOperator_SUM, Register_SUM(), ParseReducer); + } + TfLiteStatus AddSvdf( const TfLiteRegistration& registration = Register_SVDF()) { return AddBuiltin(BuiltinOperator_SVDF, registration, ParseSvdf); @@ -554,10 +559,9 @@ class MicroMutableOpResolver : public MicroOpResolver { } TfLiteStatus AddUnidirectionalSequenceLSTM() { - return AddBuiltin( - BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_LSTM, - tflite::ops::micro::Register_UNIDIRECTIONAL_SEQUENCE_LSTM(), - ParseUnidirectionalSequenceLSTM); + return AddBuiltin(BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_LSTM, + Register_UNIDIRECTIONAL_SEQUENCE_LSTM(), + ParseUnidirectionalSequenceLSTM); } TfLiteStatus AddVarHandle() { @@ -581,31 +585,20 @@ class MicroMutableOpResolver : public MicroOpResolver { const TfLiteRegistration& registration, MicroOpResolver::BuiltinParseFunction parser) { if (op == BuiltinOperator_CUSTOM) { - if (error_reporter_ != nullptr) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Invalid parameter BuiltinOperator_CUSTOM to the " - "AddBuiltin function."); - } + MicroPrintf("Invalid parameter BuiltinOperator_CUSTOM to the "); + MicroPrintf("AddBuiltin function."); return kTfLiteError; } if (FindOp(op) != nullptr) { - if (error_reporter_ != nullptr) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Calling AddBuiltin with the same op more than " - "once is not supported (Op: #%d).", - op); - } + MicroPrintf("Calling AddBuiltin with the same op more than "); + MicroPrintf("once is not supported (Op: #%d).", op); return kTfLiteError; } if (registrations_len_ >= tOpCount) { - if (error_reporter_) { - TF_LITE_REPORT_ERROR(error_reporter_, - "Couldn't register builtin op #%d, resolver size " - "is too small (%d).", - op, tOpCount); - } + MicroPrintf("Couldn't register builtin op #%d, resolver size ", op); + MicroPrintf("is too small (%d).", tOpCount); return kTfLiteError; } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_profiler.cc b/code/components/tflite-lib/tensorflow/lite/micro/micro_profiler.cc index d8a86c6b..72f3d37f 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_profiler.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_profiler.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/micro/micro_profiler.h" +#include #include #include "tensorflow/lite/kernels/internal/compatibility.h" @@ -38,7 +39,7 @@ void MicroProfiler::EndEvent(uint32_t event_handle) { end_ticks_[event_handle] = GetCurrentTimeTicks(); } -int32_t MicroProfiler::GetTotalTicks() const { +uint32_t MicroProfiler::GetTotalTicks() const { int32_t ticks = 0; for (int i = 0; i < num_events_; ++i) { ticks += end_ticks_[i] - start_ticks_[i]; @@ -49,8 +50,9 @@ int32_t MicroProfiler::GetTotalTicks() const { void MicroProfiler::Log() const { #if !defined(TF_LITE_STRIP_ERROR_STRINGS) for (int i = 0; i < num_events_; ++i) { - int32_t ticks = end_ticks_[i] - start_ticks_[i]; - MicroPrintf("%s took %d ticks (%d ms).", tags_[i], ticks, TicksToMs(ticks)); + uint32_t ticks = end_ticks_[i] - start_ticks_[i]; + MicroPrintf("%s took %" PRIu32 " ticks (%d ms).", tags_[i], ticks, + TicksToMs(ticks)); } #endif } @@ -59,8 +61,8 @@ void MicroProfiler::LogCsv() const { #if !defined(TF_LITE_STRIP_ERROR_STRINGS) MicroPrintf("\"Event\",\"Tag\",\"Ticks\""); for (int i = 0; i < num_events_; ++i) { - int32_t ticks = end_ticks_[i] - start_ticks_[i]; - MicroPrintf("%d,%s,%d", i, tags_[i], ticks); + uint32_t ticks = end_ticks_[i] - start_ticks_[i]; + MicroPrintf("%d,%s,%" PRIu32, i, tags_[i], ticks); } #endif } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_profiler.h b/code/components/tflite-lib/tensorflow/lite/micro/micro_profiler.h index 8a1ba5de..41f41a35 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_profiler.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_profiler.h @@ -51,7 +51,7 @@ class MicroProfiler { // Returns the sum of the ticks taken across all the events. This number // is only meaningful if all of the events are disjoint (the end time of // event[i] <= start time of event[i+1]). - int32_t GetTotalTicks() const; + uint32_t GetTotalTicks() const; // Prints the profiling information of each of the events in human readable // form. @@ -68,8 +68,8 @@ class MicroProfiler { static constexpr int kMaxEvents = 1024; const char* tags_[kMaxEvents]; - int32_t start_ticks_[kMaxEvents]; - int32_t end_ticks_[kMaxEvents]; + uint32_t start_ticks_[kMaxEvents]; + uint32_t end_ticks_[kMaxEvents]; int num_events_ = 0; TF_LITE_REMOVE_VIRTUAL_DELETE; diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_time.cc b/code/components/tflite-lib/tensorflow/lite/micro/micro_time.cc index bbe3f1a8..2d74fdba 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_time.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_time.cc @@ -38,21 +38,21 @@ namespace tflite { // for a platform to support Tensorflow Lite for Microcontrollers profiling. // This returns 0 by default because timing is an optional feature that builds // without errors on platforms that do not need it. -int32_t ticks_per_second() { return 0; } +uint32_t ticks_per_second() { return 0; } // Reference implementation of the GetCurrentTimeTicks() function that's // required for a platform to support Tensorflow Lite for Microcontrollers // profiling. This returns 0 by default because timing is an optional feature // that builds without errors on platforms that do not need it. -int32_t GetCurrentTimeTicks() { return 0; } +uint32_t GetCurrentTimeTicks() { return 0; } #else // defined(TF_LITE_USE_CTIME) // For platforms that support ctime, we implment the micro_time interface in // this central location. -int32_t ticks_per_second() { return CLOCKS_PER_SEC; } +uint32_t ticks_per_second() { return CLOCKS_PER_SEC; } -int32_t GetCurrentTimeTicks() { return clock(); } +uint32_t GetCurrentTimeTicks() { return clock(); } #endif } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/micro_time.h b/code/components/tflite-lib/tensorflow/lite/micro/micro_time.h index fac9069b..7a8ab455 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/micro_time.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/micro_time.h @@ -21,14 +21,14 @@ namespace tflite { // These functions should be implemented by each target platform, and provide an // accurate tick count along with how many ticks there are per second. -int32_t ticks_per_second(); +uint32_t ticks_per_second(); // Return time in ticks. The meaning of a tick varies per platform. -int32_t GetCurrentTimeTicks(); +uint32_t GetCurrentTimeTicks(); -inline int32_t TicksToMs(int32_t ticks) { - return static_cast(1000.0f * static_cast(ticks) / - static_cast(ticks_per_second())); +inline uint32_t TicksToMs(int32_t ticks) { + return static_cast(1000.0f * static_cast(ticks) / + static_cast(ticks_per_second())); } } // namespace tflite diff --git a/code/components/tflite-lib/tensorflow/lite/micro/mock_micro_graph.cc b/code/components/tflite-lib/tensorflow/lite/micro/mock_micro_graph.cc index bfd7605d..438a4065 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/mock_micro_graph.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/mock_micro_graph.cc @@ -19,7 +19,7 @@ limitations under the License. namespace tflite { -MockMicroGraph::MockMicroGraph(SimpleMemoryAllocator* allocator) +MockMicroGraph::MockMicroGraph(SingleArenaBufferAllocator* allocator) : MicroGraph(nullptr, nullptr, nullptr, nullptr), allocator_(allocator), init_count_(0), diff --git a/code/components/tflite-lib/tensorflow/lite/micro/mock_micro_graph.h b/code/components/tflite-lib/tensorflow/lite/micro/mock_micro_graph.h index c908b49e..3ae7d7cf 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/mock_micro_graph.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/mock_micro_graph.h @@ -28,7 +28,7 @@ namespace tflite { // validation of control flow operators. class MockMicroGraph : public MicroGraph { public: - explicit MockMicroGraph(SimpleMemoryAllocator* allocator); + explicit MockMicroGraph(SingleArenaBufferAllocator* allocator); TfLiteStatus InvokeSubgraph(int subgraph_idx) override; TfLiteStatus ResetVariableTensors() override; size_t NumSubgraphInputs(int subgraph_idx) override; @@ -46,7 +46,7 @@ class MockMicroGraph : public MicroGraph { private: static constexpr int kMaxSubgraphs = 10; - SimpleMemoryAllocator* allocator_; + SingleArenaBufferAllocator* allocator_; TfLiteEvalTensor* mock_tensor_; int init_count_; int prepare_count_; diff --git a/code/components/tflite-lib/tensorflow/lite/micro/recording_micro_allocator.cc b/code/components/tflite-lib/tensorflow/lite/micro/recording_micro_allocator.cc index 53b3806d..61f7816a 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/recording_micro_allocator.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/recording_micro_allocator.cc @@ -17,34 +17,34 @@ limitations under the License. #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.h" #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/memory_helpers.h" #include "tensorflow/lite/micro/memory_planner/greedy_memory_planner.h" #include "tensorflow/lite/micro/micro_allocator.h" #include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/recording_simple_memory_allocator.h" namespace tflite { size_t RecordingMicroAllocator::GetDefaultTailUsage() { // RecordingMicroAllocator inherits from MicroAllocator and its tail usage is - // similar with MicroAllocator with SimpleMemoryAllocator and MicroAllocator - // being replaced. + // similar with MicroAllocator with SingleArenaBufferAllocator and + // MicroAllocator being replaced. // TODO(b/208703041): a template version of AlignSizeUp to make expression // shorter. return MicroAllocator::GetDefaultTailUsage( /*is_memory_planner_given=*/false) + - AlignSizeUp(sizeof(RecordingSimpleMemoryAllocator), - alignof(RecordingSimpleMemoryAllocator)) - - AlignSizeUp(sizeof(SimpleMemoryAllocator), - alignof(SimpleMemoryAllocator)) + + AlignSizeUp(sizeof(RecordingSingleArenaBufferAllocator), + alignof(RecordingSingleArenaBufferAllocator)) - + AlignSizeUp(sizeof(SingleArenaBufferAllocator), + alignof(SingleArenaBufferAllocator)) + AlignSizeUp(sizeof(RecordingMicroAllocator), alignof(RecordingMicroAllocator)) - AlignSizeUp(sizeof(MicroAllocator), alignof(MicroAllocator)); } RecordingMicroAllocator::RecordingMicroAllocator( - RecordingSimpleMemoryAllocator* recording_memory_allocator, + RecordingSingleArenaBufferAllocator* recording_memory_allocator, MicroMemoryPlanner* memory_planner, ErrorReporter* error_reporter) : MicroAllocator(recording_memory_allocator, memory_planner, error_reporter), @@ -54,9 +54,9 @@ RecordingMicroAllocator* RecordingMicroAllocator::Create( uint8_t* tensor_arena, size_t arena_size, ErrorReporter* error_reporter) { TFLITE_DCHECK(error_reporter != nullptr); - RecordingSimpleMemoryAllocator* simple_memory_allocator = - RecordingSimpleMemoryAllocator::Create(error_reporter, tensor_arena, - arena_size); + RecordingSingleArenaBufferAllocator* simple_memory_allocator = + RecordingSingleArenaBufferAllocator::Create(error_reporter, tensor_arena, + arena_size); TFLITE_DCHECK(simple_memory_allocator != nullptr); uint8_t* memory_planner_buffer = @@ -96,7 +96,7 @@ RecordedAllocation RecordingMicroAllocator::GetRecordedAllocation( return RecordedAllocation(); } -const RecordingSimpleMemoryAllocator* +const RecordingSingleArenaBufferAllocator* RecordingMicroAllocator::GetSimpleMemoryAllocator() const { return recording_memory_allocator_; } @@ -167,15 +167,15 @@ TfLiteStatus RecordingMicroAllocator::AllocateNodeAndRegistrations( subgraph_idx++) { RecordAllocationUsage(allocations, recorded_node_and_registration_array_data_); - // The allocation count in SimpleMemoryAllocator will only be 1. To provide - // better logging, decrement by 1 and add in the actual number of operators - // used in the graph: - // The allocation for this recording will always be 1. This is because the - // parent class mallocs one large allocation for the number of nodes in the - // graph (e.g. sizeof(NodeAndRegistration) * num_nodes). - // To prevent extra overhead and potential for fragmentation, manually - // adjust the accounting by decrementing by 1 and adding the actual number - // of nodes used in the graph: + // The allocation count in SingleArenaBufferAllocator will only be 1. To + // provide better logging, decrement by 1 and add in the actual number of + // operators used in the graph: The allocation for this recording will + // always be 1. This is because the parent class mallocs one large + // allocation for the number of nodes in the graph (e.g. + // sizeof(NodeAndRegistration) * num_nodes). To prevent extra overhead and + // potential for fragmentation, manually adjust the accounting by + // decrementing by 1 and adding the actual number of nodes used in the + // graph: recorded_node_and_registration_array_data_.count += model->subgraphs()->Get(subgraph_idx)->operators()->size() - 1; } diff --git a/code/components/tflite-lib/tensorflow/lite/micro/recording_micro_allocator.h b/code/components/tflite-lib/tensorflow/lite/micro/recording_micro_allocator.h index 6b039c03..699b1a22 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/recording_micro_allocator.h +++ b/code/components/tflite-lib/tensorflow/lite/micro/recording_micro_allocator.h @@ -16,9 +16,9 @@ limitations under the License. #ifndef TENSORFLOW_LITE_MICRO_RECORDING_MICRO_ALLOCATOR_H_ #define TENSORFLOW_LITE_MICRO_RECORDING_MICRO_ALLOCATOR_H_ +#include "tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.h" #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/micro_allocator.h" -#include "tensorflow/lite/micro/recording_simple_memory_allocator.h" namespace tflite { @@ -48,10 +48,10 @@ struct RecordedAllocation { // Utility subclass of MicroAllocator that records all allocations // inside the arena. A summary of allocations can be logged through the // ErrorReporter by invoking LogAllocations(). This special allocator requires -// an instance of RecordingSimpleMemoryAllocator to capture allocations in the -// head and tail. Arena allocation recording can be retrieved by type through -// the GetRecordedAllocation() function. This class should only be used for -// auditing memory usage or integration testing. +// an instance of RecordingSingleArenaBufferAllocator to capture allocations in +// the head and tail. Arena allocation recording can be retrieved by type +// through the GetRecordedAllocation() function. This class should only be used +// for auditing memory usage or integration testing. class RecordingMicroAllocator : public MicroAllocator { public: static RecordingMicroAllocator* Create(uint8_t* tensor_arena, @@ -65,7 +65,7 @@ class RecordingMicroAllocator : public MicroAllocator { RecordedAllocation GetRecordedAllocation( RecordedAllocationType allocation_type) const; - const RecordingSimpleMemoryAllocator* GetSimpleMemoryAllocator() const; + const RecordingSingleArenaBufferAllocator* GetSimpleMemoryAllocator() const; // Logs out through the ErrorReporter all allocation recordings by type // defined in RecordedAllocationType. @@ -94,7 +94,7 @@ class RecordingMicroAllocator : public MicroAllocator { bool allocate_temp) override; private: - RecordingMicroAllocator(RecordingSimpleMemoryAllocator* memory_allocator, + RecordingMicroAllocator(RecordingSingleArenaBufferAllocator* memory_allocator, MicroMemoryPlanner* memory_planner, ErrorReporter* error_reporter); @@ -106,7 +106,7 @@ class RecordingMicroAllocator : public MicroAllocator { void RecordAllocationUsage(const RecordedAllocation& snapshotted_allocation, RecordedAllocation& recorded_allocation); - const RecordingSimpleMemoryAllocator* recording_memory_allocator_; + const RecordingSingleArenaBufferAllocator* recording_memory_allocator_; RecordedAllocation recorded_tflite_eval_tensor_data_ = {}; RecordedAllocation recorded_persistent_tflite_tensor_data_ = {}; diff --git a/code/components/tflite-lib/tensorflow/lite/micro/test_helpers.cc b/code/components/tflite-lib/tensorflow/lite/micro/test_helpers.cc index 2adea777..2411bbf8 100644 --- a/code/components/tflite-lib/tensorflow/lite/micro/test_helpers.cc +++ b/code/components/tflite-lib/tensorflow/lite/micro/test_helpers.cc @@ -110,7 +110,9 @@ class ModelBuilder { // Adds a node to the model with given input and output Tensors. Node AddNode(Operator op, std::initializer_list inputs, - std::initializer_list outputs); + std::initializer_list outputs, + std::initializer_list intermediates = + std::initializer_list{}); void AddMetadata(const char* description_string, const int32_t* metadata_buffer_data, size_t num_elements); @@ -165,12 +167,17 @@ ModelBuilder::Operator ModelBuilder::RegisterOp(BuiltinOperator op, ModelBuilder::Node ModelBuilder::AddNode( ModelBuilder::Operator op, std::initializer_list inputs, - std::initializer_list outputs) { + std::initializer_list outputs, + std::initializer_list intermediates) { TFLITE_DCHECK(next_operator_id_ <= kMaxOperators); operators_[next_operator_id_] = tflite::CreateOperator( *builder_, op, builder_->CreateVector(inputs.begin(), inputs.size()), builder_->CreateVector(outputs.begin(), outputs.size()), - BuiltinOptions_NONE); + BuiltinOptions_NONE, + /*builtin_options=*/0, + /*custom_options=*/0, tflite::CustomOptionsFormat_FLEXBUFFERS, + /*mutating_variable_inputs =*/0, + builder_->CreateVector(intermediates.begin(), intermediates.size())); next_operator_id_++; return next_operator_id_ - 1; } @@ -274,9 +281,12 @@ const Model* BuildSimpleStatefulModel() { const int median_tensor = model_builder.AddTensor(TensorType_INT8, {3}); const int invoke_count_tensor = model_builder.AddTensor(TensorType_INT32, {1}); + const int intermediate_tensor = + model_builder.AddTensor(TensorType_FLOAT32, {0}); model_builder.AddNode(op_id, {input_tensor}, - {median_tensor, invoke_count_tensor}); + {median_tensor, invoke_count_tensor}, + {intermediate_tensor}); return model_builder.BuildModel({input_tensor}, {median_tensor, invoke_count_tensor}); } diff --git a/code/components/tflite-lib/tensorflow/lite/schema/schema_generated.h b/code/components/tflite-lib/tensorflow/lite/schema/schema_generated.h index e5ce189f..5d1cee85 100644 --- a/code/components/tflite-lib/tensorflow/lite/schema/schema_generated.h +++ b/code/components/tflite-lib/tensorflow/lite/schema/schema_generated.h @@ -1,17 +1,3 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ // automatically generated by the FlatBuffers compiler, do not modify @@ -23,402 +9,555 @@ limitations under the License. namespace tflite { struct CustomQuantization; +struct CustomQuantizationBuilder; struct CustomQuantizationT; struct QuantizationParameters; +struct QuantizationParametersBuilder; struct QuantizationParametersT; struct Int32Vector; +struct Int32VectorBuilder; struct Int32VectorT; struct Uint16Vector; +struct Uint16VectorBuilder; struct Uint16VectorT; struct Uint8Vector; +struct Uint8VectorBuilder; struct Uint8VectorT; struct DimensionMetadata; +struct DimensionMetadataBuilder; struct DimensionMetadataT; struct SparsityParameters; +struct SparsityParametersBuilder; struct SparsityParametersT; struct Tensor; +struct TensorBuilder; struct TensorT; struct Conv2DOptions; +struct Conv2DOptionsBuilder; struct Conv2DOptionsT; struct Conv3DOptions; +struct Conv3DOptionsBuilder; struct Conv3DOptionsT; struct Pool2DOptions; +struct Pool2DOptionsBuilder; struct Pool2DOptionsT; struct DepthwiseConv2DOptions; +struct DepthwiseConv2DOptionsBuilder; struct DepthwiseConv2DOptionsT; struct ConcatEmbeddingsOptions; +struct ConcatEmbeddingsOptionsBuilder; struct ConcatEmbeddingsOptionsT; struct LSHProjectionOptions; +struct LSHProjectionOptionsBuilder; struct LSHProjectionOptionsT; struct SVDFOptions; +struct SVDFOptionsBuilder; struct SVDFOptionsT; struct RNNOptions; +struct RNNOptionsBuilder; struct RNNOptionsT; struct SequenceRNNOptions; +struct SequenceRNNOptionsBuilder; struct SequenceRNNOptionsT; struct BidirectionalSequenceRNNOptions; +struct BidirectionalSequenceRNNOptionsBuilder; struct BidirectionalSequenceRNNOptionsT; struct FullyConnectedOptions; +struct FullyConnectedOptionsBuilder; struct FullyConnectedOptionsT; struct SoftmaxOptions; +struct SoftmaxOptionsBuilder; struct SoftmaxOptionsT; struct ConcatenationOptions; +struct ConcatenationOptionsBuilder; struct ConcatenationOptionsT; struct AddOptions; +struct AddOptionsBuilder; struct AddOptionsT; struct MulOptions; +struct MulOptionsBuilder; struct MulOptionsT; struct L2NormOptions; +struct L2NormOptionsBuilder; struct L2NormOptionsT; struct LocalResponseNormalizationOptions; +struct LocalResponseNormalizationOptionsBuilder; struct LocalResponseNormalizationOptionsT; struct LSTMOptions; +struct LSTMOptionsBuilder; struct LSTMOptionsT; struct UnidirectionalSequenceLSTMOptions; +struct UnidirectionalSequenceLSTMOptionsBuilder; struct UnidirectionalSequenceLSTMOptionsT; struct BidirectionalSequenceLSTMOptions; +struct BidirectionalSequenceLSTMOptionsBuilder; struct BidirectionalSequenceLSTMOptionsT; struct ResizeBilinearOptions; +struct ResizeBilinearOptionsBuilder; struct ResizeBilinearOptionsT; struct ResizeNearestNeighborOptions; +struct ResizeNearestNeighborOptionsBuilder; struct ResizeNearestNeighborOptionsT; struct CallOptions; +struct CallOptionsBuilder; struct CallOptionsT; struct PadOptions; +struct PadOptionsBuilder; struct PadOptionsT; struct PadV2Options; +struct PadV2OptionsBuilder; struct PadV2OptionsT; struct ReshapeOptions; +struct ReshapeOptionsBuilder; struct ReshapeOptionsT; struct SpaceToBatchNDOptions; +struct SpaceToBatchNDOptionsBuilder; struct SpaceToBatchNDOptionsT; struct BatchToSpaceNDOptions; +struct BatchToSpaceNDOptionsBuilder; struct BatchToSpaceNDOptionsT; struct SkipGramOptions; +struct SkipGramOptionsBuilder; struct SkipGramOptionsT; struct SpaceToDepthOptions; +struct SpaceToDepthOptionsBuilder; struct SpaceToDepthOptionsT; struct DepthToSpaceOptions; +struct DepthToSpaceOptionsBuilder; struct DepthToSpaceOptionsT; struct SubOptions; +struct SubOptionsBuilder; struct SubOptionsT; struct DivOptions; +struct DivOptionsBuilder; struct DivOptionsT; struct TopKV2Options; +struct TopKV2OptionsBuilder; struct TopKV2OptionsT; struct EmbeddingLookupSparseOptions; +struct EmbeddingLookupSparseOptionsBuilder; struct EmbeddingLookupSparseOptionsT; struct GatherOptions; +struct GatherOptionsBuilder; struct GatherOptionsT; struct TransposeOptions; +struct TransposeOptionsBuilder; struct TransposeOptionsT; struct ExpOptions; +struct ExpOptionsBuilder; struct ExpOptionsT; struct CosOptions; +struct CosOptionsBuilder; struct CosOptionsT; struct ReducerOptions; +struct ReducerOptionsBuilder; struct ReducerOptionsT; struct SqueezeOptions; +struct SqueezeOptionsBuilder; struct SqueezeOptionsT; struct SplitOptions; +struct SplitOptionsBuilder; struct SplitOptionsT; struct SplitVOptions; +struct SplitVOptionsBuilder; struct SplitVOptionsT; struct StridedSliceOptions; +struct StridedSliceOptionsBuilder; struct StridedSliceOptionsT; struct LogSoftmaxOptions; +struct LogSoftmaxOptionsBuilder; struct LogSoftmaxOptionsT; struct CastOptions; +struct CastOptionsBuilder; struct CastOptionsT; struct DequantizeOptions; +struct DequantizeOptionsBuilder; struct DequantizeOptionsT; struct MaximumMinimumOptions; +struct MaximumMinimumOptionsBuilder; struct MaximumMinimumOptionsT; struct TileOptions; +struct TileOptionsBuilder; struct TileOptionsT; struct ArgMaxOptions; +struct ArgMaxOptionsBuilder; struct ArgMaxOptionsT; struct ArgMinOptions; +struct ArgMinOptionsBuilder; struct ArgMinOptionsT; struct GreaterOptions; +struct GreaterOptionsBuilder; struct GreaterOptionsT; struct GreaterEqualOptions; +struct GreaterEqualOptionsBuilder; struct GreaterEqualOptionsT; struct LessOptions; +struct LessOptionsBuilder; struct LessOptionsT; struct LessEqualOptions; +struct LessEqualOptionsBuilder; struct LessEqualOptionsT; struct NegOptions; +struct NegOptionsBuilder; struct NegOptionsT; struct SelectOptions; +struct SelectOptionsBuilder; struct SelectOptionsT; struct SliceOptions; +struct SliceOptionsBuilder; struct SliceOptionsT; struct TransposeConvOptions; +struct TransposeConvOptionsBuilder; struct TransposeConvOptionsT; struct ExpandDimsOptions; +struct ExpandDimsOptionsBuilder; struct ExpandDimsOptionsT; struct SparseToDenseOptions; +struct SparseToDenseOptionsBuilder; struct SparseToDenseOptionsT; struct EqualOptions; +struct EqualOptionsBuilder; struct EqualOptionsT; struct NotEqualOptions; +struct NotEqualOptionsBuilder; struct NotEqualOptionsT; struct ShapeOptions; +struct ShapeOptionsBuilder; struct ShapeOptionsT; struct RankOptions; +struct RankOptionsBuilder; struct RankOptionsT; struct PowOptions; +struct PowOptionsBuilder; struct PowOptionsT; struct FakeQuantOptions; +struct FakeQuantOptionsBuilder; struct FakeQuantOptionsT; struct PackOptions; +struct PackOptionsBuilder; struct PackOptionsT; struct LogicalOrOptions; +struct LogicalOrOptionsBuilder; struct LogicalOrOptionsT; struct OneHotOptions; +struct OneHotOptionsBuilder; struct OneHotOptionsT; struct AbsOptions; +struct AbsOptionsBuilder; struct AbsOptionsT; struct HardSwishOptions; +struct HardSwishOptionsBuilder; struct HardSwishOptionsT; struct LogicalAndOptions; +struct LogicalAndOptionsBuilder; struct LogicalAndOptionsT; struct LogicalNotOptions; +struct LogicalNotOptionsBuilder; struct LogicalNotOptionsT; struct UnpackOptions; +struct UnpackOptionsBuilder; struct UnpackOptionsT; struct FloorDivOptions; +struct FloorDivOptionsBuilder; struct FloorDivOptionsT; struct SquareOptions; +struct SquareOptionsBuilder; struct SquareOptionsT; struct ZerosLikeOptions; +struct ZerosLikeOptionsBuilder; struct ZerosLikeOptionsT; struct FillOptions; +struct FillOptionsBuilder; struct FillOptionsT; struct FloorModOptions; +struct FloorModOptionsBuilder; struct FloorModOptionsT; struct RangeOptions; +struct RangeOptionsBuilder; struct RangeOptionsT; struct LeakyReluOptions; +struct LeakyReluOptionsBuilder; struct LeakyReluOptionsT; struct SquaredDifferenceOptions; +struct SquaredDifferenceOptionsBuilder; struct SquaredDifferenceOptionsT; struct MirrorPadOptions; +struct MirrorPadOptionsBuilder; struct MirrorPadOptionsT; struct UniqueOptions; +struct UniqueOptionsBuilder; struct UniqueOptionsT; struct ReverseV2Options; +struct ReverseV2OptionsBuilder; struct ReverseV2OptionsT; struct AddNOptions; +struct AddNOptionsBuilder; struct AddNOptionsT; struct GatherNdOptions; +struct GatherNdOptionsBuilder; struct GatherNdOptionsT; struct WhereOptions; +struct WhereOptionsBuilder; struct WhereOptionsT; struct ReverseSequenceOptions; +struct ReverseSequenceOptionsBuilder; struct ReverseSequenceOptionsT; struct MatrixDiagOptions; +struct MatrixDiagOptionsBuilder; struct MatrixDiagOptionsT; struct QuantizeOptions; +struct QuantizeOptionsBuilder; struct QuantizeOptionsT; struct MatrixSetDiagOptions; +struct MatrixSetDiagOptionsBuilder; struct MatrixSetDiagOptionsT; struct IfOptions; +struct IfOptionsBuilder; struct IfOptionsT; struct CallOnceOptions; +struct CallOnceOptionsBuilder; struct CallOnceOptionsT; struct WhileOptions; +struct WhileOptionsBuilder; struct WhileOptionsT; struct NonMaxSuppressionV4Options; +struct NonMaxSuppressionV4OptionsBuilder; struct NonMaxSuppressionV4OptionsT; struct NonMaxSuppressionV5Options; +struct NonMaxSuppressionV5OptionsBuilder; struct NonMaxSuppressionV5OptionsT; struct ScatterNdOptions; +struct ScatterNdOptionsBuilder; struct ScatterNdOptionsT; struct SelectV2Options; +struct SelectV2OptionsBuilder; struct SelectV2OptionsT; struct DensifyOptions; +struct DensifyOptionsBuilder; struct DensifyOptionsT; struct SegmentSumOptions; +struct SegmentSumOptionsBuilder; struct SegmentSumOptionsT; struct BatchMatMulOptions; +struct BatchMatMulOptionsBuilder; struct BatchMatMulOptionsT; struct CumsumOptions; +struct CumsumOptionsBuilder; struct CumsumOptionsT; struct BroadcastToOptions; +struct BroadcastToOptionsBuilder; struct BroadcastToOptionsT; struct Rfft2dOptions; +struct Rfft2dOptionsBuilder; struct Rfft2dOptionsT; struct HashtableOptions; +struct HashtableOptionsBuilder; struct HashtableOptionsT; struct HashtableFindOptions; +struct HashtableFindOptionsBuilder; struct HashtableFindOptionsT; struct HashtableImportOptions; +struct HashtableImportOptionsBuilder; struct HashtableImportOptionsT; struct HashtableSizeOptions; +struct HashtableSizeOptionsBuilder; struct HashtableSizeOptionsT; struct VarHandleOptions; +struct VarHandleOptionsBuilder; struct VarHandleOptionsT; struct ReadVariableOptions; +struct ReadVariableOptionsBuilder; struct ReadVariableOptionsT; struct AssignVariableOptions; +struct AssignVariableOptionsBuilder; struct AssignVariableOptionsT; struct RandomOptions; +struct RandomOptionsBuilder; struct RandomOptionsT; struct BucketizeOptions; +struct BucketizeOptionsBuilder; struct BucketizeOptionsT; struct GeluOptions; +struct GeluOptionsBuilder; struct GeluOptionsT; struct DynamicUpdateSliceOptions; +struct DynamicUpdateSliceOptionsBuilder; struct DynamicUpdateSliceOptionsT; +struct UnsortedSegmentProdOptions; +struct UnsortedSegmentProdOptionsBuilder; +struct UnsortedSegmentProdOptionsT; + +struct UnsortedSegmentMaxOptions; +struct UnsortedSegmentMaxOptionsBuilder; +struct UnsortedSegmentMaxOptionsT; + +struct UnsortedSegmentSumOptions; +struct UnsortedSegmentSumOptionsBuilder; +struct UnsortedSegmentSumOptionsT; + +struct ATan2Options; +struct ATan2OptionsBuilder; +struct ATan2OptionsT; + +struct UnsortedSegmentMinOptions; +struct UnsortedSegmentMinOptionsBuilder; +struct UnsortedSegmentMinOptionsT; + struct OperatorCode; +struct OperatorCodeBuilder; struct OperatorCodeT; struct Operator; +struct OperatorBuilder; struct OperatorT; struct SubGraph; +struct SubGraphBuilder; struct SubGraphT; struct Buffer; +struct BufferBuilder; struct BufferT; struct Metadata; +struct MetadataBuilder; struct MetadataT; struct TensorMap; +struct TensorMapBuilder; struct TensorMapT; struct SignatureDef; +struct SignatureDefBuilder; struct SignatureDefT; struct Model; +struct ModelBuilder; struct ModelT; enum TensorType { @@ -542,8 +681,8 @@ struct QuantizationDetailsUnion { QuantizationDetailsUnion(QuantizationDetailsUnion&& u) FLATBUFFERS_NOEXCEPT : type(QuantizationDetails_NONE), value(nullptr) { std::swap(type, u.type); std::swap(value, u.value); } - QuantizationDetailsUnion(const QuantizationDetailsUnion &) FLATBUFFERS_NOEXCEPT; - QuantizationDetailsUnion &operator=(const QuantizationDetailsUnion &u) FLATBUFFERS_NOEXCEPT + QuantizationDetailsUnion(const QuantizationDetailsUnion &); + QuantizationDetailsUnion &operator=(const QuantizationDetailsUnion &u) { QuantizationDetailsUnion t(u); std::swap(type, t.type); std::swap(value, t.value); return *this; } QuantizationDetailsUnion &operator=(QuantizationDetailsUnion &&u) FLATBUFFERS_NOEXCEPT { std::swap(type, u.type); std::swap(value, u.value); return *this; } @@ -669,8 +808,8 @@ struct SparseIndexVectorUnion { SparseIndexVectorUnion(SparseIndexVectorUnion&& u) FLATBUFFERS_NOEXCEPT : type(SparseIndexVector_NONE), value(nullptr) { std::swap(type, u.type); std::swap(value, u.value); } - SparseIndexVectorUnion(const SparseIndexVectorUnion &) FLATBUFFERS_NOEXCEPT; - SparseIndexVectorUnion &operator=(const SparseIndexVectorUnion &u) FLATBUFFERS_NOEXCEPT + SparseIndexVectorUnion(const SparseIndexVectorUnion &); + SparseIndexVectorUnion &operator=(const SparseIndexVectorUnion &u) { SparseIndexVectorUnion t(u); std::swap(type, t.type); std::swap(value, t.value); return *this; } SparseIndexVectorUnion &operator=(SparseIndexVectorUnion &&u) FLATBUFFERS_NOEXCEPT { std::swap(type, u.type); std::swap(value, u.value); return *this; } @@ -875,11 +1014,17 @@ enum BuiltinOperator { BuiltinOperator_MULTINOMIAL = 149, BuiltinOperator_GELU = 150, BuiltinOperator_DYNAMIC_UPDATE_SLICE = 151, + BuiltinOperator_RELU_0_TO_1 = 152, + BuiltinOperator_UNSORTED_SEGMENT_PROD = 153, + BuiltinOperator_UNSORTED_SEGMENT_MAX = 154, + BuiltinOperator_UNSORTED_SEGMENT_SUM = 155, + BuiltinOperator_ATAN2 = 156, + BuiltinOperator_UNSORTED_SEGMENT_MIN = 157, BuiltinOperator_MIN = BuiltinOperator_ADD, - BuiltinOperator_MAX = BuiltinOperator_DYNAMIC_UPDATE_SLICE + BuiltinOperator_MAX = BuiltinOperator_UNSORTED_SEGMENT_MIN }; -inline const BuiltinOperator (&EnumValuesBuiltinOperator())[152] { +inline const BuiltinOperator (&EnumValuesBuiltinOperator())[158] { static const BuiltinOperator values[] = { BuiltinOperator_ADD, BuiltinOperator_AVERAGE_POOL_2D, @@ -1032,13 +1177,19 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[152] { BuiltinOperator_RANDOM_UNIFORM, BuiltinOperator_MULTINOMIAL, BuiltinOperator_GELU, - BuiltinOperator_DYNAMIC_UPDATE_SLICE + BuiltinOperator_DYNAMIC_UPDATE_SLICE, + BuiltinOperator_RELU_0_TO_1, + BuiltinOperator_UNSORTED_SEGMENT_PROD, + BuiltinOperator_UNSORTED_SEGMENT_MAX, + BuiltinOperator_UNSORTED_SEGMENT_SUM, + BuiltinOperator_ATAN2, + BuiltinOperator_UNSORTED_SEGMENT_MIN }; return values; } inline const char * const *EnumNamesBuiltinOperator() { - static const char * const names[153] = { + static const char * const names[159] = { "ADD", "AVERAGE_POOL_2D", "CONCATENATION", @@ -1191,13 +1342,19 @@ inline const char * const *EnumNamesBuiltinOperator() { "MULTINOMIAL", "GELU", "DYNAMIC_UPDATE_SLICE", + "RELU_0_TO_1", + "UNSORTED_SEGMENT_PROD", + "UNSORTED_SEGMENT_MAX", + "UNSORTED_SEGMENT_SUM", + "ATAN2", + "UNSORTED_SEGMENT_MIN", nullptr }; return names; } inline const char *EnumNameBuiltinOperator(BuiltinOperator e) { - if (flatbuffers::IsOutRange(e, BuiltinOperator_ADD, BuiltinOperator_DYNAMIC_UPDATE_SLICE)) return ""; + if (flatbuffers::IsOutRange(e, BuiltinOperator_ADD, BuiltinOperator_UNSORTED_SEGMENT_MIN)) return ""; const size_t index = static_cast(e); return EnumNamesBuiltinOperator()[index]; } @@ -1321,11 +1478,16 @@ enum BuiltinOptions { BuiltinOptions_BucketizeOptions = 115, BuiltinOptions_GeluOptions = 116, BuiltinOptions_DynamicUpdateSliceOptions = 117, + BuiltinOptions_UnsortedSegmentProdOptions = 118, + BuiltinOptions_UnsortedSegmentMaxOptions = 119, + BuiltinOptions_UnsortedSegmentMinOptions = 120, + BuiltinOptions_UnsortedSegmentSumOptions = 121, + BuiltinOptions_ATan2Options = 122, BuiltinOptions_MIN = BuiltinOptions_NONE, - BuiltinOptions_MAX = BuiltinOptions_DynamicUpdateSliceOptions + BuiltinOptions_MAX = BuiltinOptions_ATan2Options }; -inline const BuiltinOptions (&EnumValuesBuiltinOptions())[118] { +inline const BuiltinOptions (&EnumValuesBuiltinOptions())[123] { static const BuiltinOptions values[] = { BuiltinOptions_NONE, BuiltinOptions_Conv2DOptions, @@ -1444,13 +1606,18 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[118] { BuiltinOptions_RandomOptions, BuiltinOptions_BucketizeOptions, BuiltinOptions_GeluOptions, - BuiltinOptions_DynamicUpdateSliceOptions + BuiltinOptions_DynamicUpdateSliceOptions, + BuiltinOptions_UnsortedSegmentProdOptions, + BuiltinOptions_UnsortedSegmentMaxOptions, + BuiltinOptions_UnsortedSegmentMinOptions, + BuiltinOptions_UnsortedSegmentSumOptions, + BuiltinOptions_ATan2Options }; return values; } inline const char * const *EnumNamesBuiltinOptions() { - static const char * const names[119] = { + static const char * const names[124] = { "NONE", "Conv2DOptions", "DepthwiseConv2DOptions", @@ -1569,13 +1736,18 @@ inline const char * const *EnumNamesBuiltinOptions() { "BucketizeOptions", "GeluOptions", "DynamicUpdateSliceOptions", + "UnsortedSegmentProdOptions", + "UnsortedSegmentMaxOptions", + "UnsortedSegmentMinOptions", + "UnsortedSegmentSumOptions", + "ATan2Options", nullptr }; return names; } inline const char *EnumNameBuiltinOptions(BuiltinOptions e) { - if (flatbuffers::IsOutRange(e, BuiltinOptions_NONE, BuiltinOptions_DynamicUpdateSliceOptions)) return ""; + if (flatbuffers::IsOutRange(e, BuiltinOptions_NONE, BuiltinOptions_ATan2Options)) return ""; const size_t index = static_cast(e); return EnumNamesBuiltinOptions()[index]; } @@ -2052,6 +2224,26 @@ template<> struct BuiltinOptionsTraits { static const BuiltinOptions enum_value = BuiltinOptions_DynamicUpdateSliceOptions; }; +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_UnsortedSegmentProdOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_UnsortedSegmentMaxOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_UnsortedSegmentMinOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_UnsortedSegmentSumOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_ATan2Options; +}; + struct BuiltinOptionsUnion { BuiltinOptions type; void *value; @@ -2060,8 +2252,8 @@ struct BuiltinOptionsUnion { BuiltinOptionsUnion(BuiltinOptionsUnion&& u) FLATBUFFERS_NOEXCEPT : type(BuiltinOptions_NONE), value(nullptr) { std::swap(type, u.type); std::swap(value, u.value); } - BuiltinOptionsUnion(const BuiltinOptionsUnion &) FLATBUFFERS_NOEXCEPT; - BuiltinOptionsUnion &operator=(const BuiltinOptionsUnion &u) FLATBUFFERS_NOEXCEPT + BuiltinOptionsUnion(const BuiltinOptionsUnion &); + BuiltinOptionsUnion &operator=(const BuiltinOptionsUnion &u) { BuiltinOptionsUnion t(u); std::swap(type, t.type); std::swap(value, t.value); return *this; } BuiltinOptionsUnion &operator=(BuiltinOptionsUnion &&u) FLATBUFFERS_NOEXCEPT { std::swap(type, u.type); std::swap(value, u.value); return *this; } @@ -3020,6 +3212,46 @@ struct BuiltinOptionsUnion { return type == BuiltinOptions_DynamicUpdateSliceOptions ? reinterpret_cast(value) : nullptr; } + tflite::UnsortedSegmentProdOptionsT *AsUnsortedSegmentProdOptions() { + return type == BuiltinOptions_UnsortedSegmentProdOptions ? + reinterpret_cast(value) : nullptr; + } + const tflite::UnsortedSegmentProdOptionsT *AsUnsortedSegmentProdOptions() const { + return type == BuiltinOptions_UnsortedSegmentProdOptions ? + reinterpret_cast(value) : nullptr; + } + tflite::UnsortedSegmentMaxOptionsT *AsUnsortedSegmentMaxOptions() { + return type == BuiltinOptions_UnsortedSegmentMaxOptions ? + reinterpret_cast(value) : nullptr; + } + const tflite::UnsortedSegmentMaxOptionsT *AsUnsortedSegmentMaxOptions() const { + return type == BuiltinOptions_UnsortedSegmentMaxOptions ? + reinterpret_cast(value) : nullptr; + } + tflite::UnsortedSegmentMinOptionsT *AsUnsortedSegmentMinOptions() { + return type == BuiltinOptions_UnsortedSegmentMinOptions ? + reinterpret_cast(value) : nullptr; + } + const tflite::UnsortedSegmentMinOptionsT *AsUnsortedSegmentMinOptions() const { + return type == BuiltinOptions_UnsortedSegmentMinOptions ? + reinterpret_cast(value) : nullptr; + } + tflite::UnsortedSegmentSumOptionsT *AsUnsortedSegmentSumOptions() { + return type == BuiltinOptions_UnsortedSegmentSumOptions ? + reinterpret_cast(value) : nullptr; + } + const tflite::UnsortedSegmentSumOptionsT *AsUnsortedSegmentSumOptions() const { + return type == BuiltinOptions_UnsortedSegmentSumOptions ? + reinterpret_cast(value) : nullptr; + } + tflite::ATan2OptionsT *AsATan2Options() { + return type == BuiltinOptions_ATan2Options ? + reinterpret_cast(value) : nullptr; + } + const tflite::ATan2OptionsT *AsATan2Options() const { + return type == BuiltinOptions_ATan2Options ? + reinterpret_cast(value) : nullptr; + } }; bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type); @@ -3289,6 +3521,7 @@ struct CustomQuantizationT : public flatbuffers::NativeTable { struct CustomQuantization FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef CustomQuantizationT NativeTableType; + typedef CustomQuantizationBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_CUSTOM = 4 }; @@ -3307,6 +3540,7 @@ struct CustomQuantization FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct CustomQuantizationBuilder { + typedef CustomQuantization Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_custom(flatbuffers::Offset> custom) { @@ -3359,6 +3593,7 @@ struct QuantizationParametersT : public flatbuffers::NativeTable { struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef QuantizationParametersT NativeTableType; + typedef QuantizationParametersBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_MIN = 4, VT_MAX = 6, @@ -3419,6 +3654,7 @@ template<> inline const tflite::CustomQuantization *QuantizationParameters::deta } struct QuantizationParametersBuilder { + typedef QuantizationParameters Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_min(flatbuffers::Offset> min) { @@ -3509,6 +3745,7 @@ struct Int32VectorT : public flatbuffers::NativeTable { struct Int32Vector FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef Int32VectorT NativeTableType; + typedef Int32VectorBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_VALUES = 4 }; @@ -3527,6 +3764,7 @@ struct Int32Vector FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct Int32VectorBuilder { + typedef Int32Vector Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_values(flatbuffers::Offset> values) { @@ -3572,6 +3810,7 @@ struct Uint16VectorT : public flatbuffers::NativeTable { struct Uint16Vector FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef Uint16VectorT NativeTableType; + typedef Uint16VectorBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_VALUES = 4 }; @@ -3590,6 +3829,7 @@ struct Uint16Vector FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct Uint16VectorBuilder { + typedef Uint16Vector Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_values(flatbuffers::Offset> values) { @@ -3636,6 +3876,7 @@ struct Uint8VectorT : public flatbuffers::NativeTable { struct Uint8Vector FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef Uint8VectorT NativeTableType; + typedef Uint8VectorBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_VALUES = 4 }; @@ -3654,6 +3895,7 @@ struct Uint8Vector FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct Uint8VectorBuilder { + typedef Uint8Vector Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_values(flatbuffers::Offset> values) { @@ -3705,6 +3947,7 @@ struct DimensionMetadataT : public flatbuffers::NativeTable { struct DimensionMetadata FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef DimensionMetadataT NativeTableType; + typedef DimensionMetadataBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FORMAT = 4, VT_DENSE_SIZE = 6, @@ -3793,6 +4036,7 @@ template<> inline const tflite::Uint8Vector *DimensionMetadata::array_indices_as } struct DimensionMetadataBuilder { + typedef DimensionMetadata Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_format(tflite::DimensionType format) { @@ -3856,6 +4100,7 @@ struct SparsityParametersT : public flatbuffers::NativeTable { struct SparsityParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SparsityParametersT NativeTableType; + typedef SparsityParametersBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_TRAVERSAL_ORDER = 4, VT_BLOCK_MAP = 6, @@ -3887,6 +4132,7 @@ struct SparsityParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SparsityParametersBuilder { + typedef SparsityParameters Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_traversal_order(flatbuffers::Offset> traversal_order) { @@ -3949,15 +4195,18 @@ struct TensorT : public flatbuffers::NativeTable { bool is_variable; std::unique_ptr sparsity; std::vector shape_signature; + bool has_rank; TensorT() : type(tflite::TensorType_FLOAT32), buffer(0), - is_variable(false) { + is_variable(false), + has_rank(false) { } }; struct Tensor FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef TensorT NativeTableType; + typedef TensorBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_SHAPE = 4, VT_TYPE = 6, @@ -3966,7 +4215,8 @@ struct Tensor FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VT_QUANTIZATION = 12, VT_IS_VARIABLE = 14, VT_SPARSITY = 16, - VT_SHAPE_SIGNATURE = 18 + VT_SHAPE_SIGNATURE = 18, + VT_HAS_RANK = 20 }; const flatbuffers::Vector *shape() const { return GetPointer *>(VT_SHAPE); @@ -3992,6 +4242,9 @@ struct Tensor FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const flatbuffers::Vector *shape_signature() const { return GetPointer *>(VT_SHAPE_SIGNATURE); } + bool has_rank() const { + return GetField(VT_HAS_RANK, 0) != 0; + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_SHAPE) && @@ -4007,6 +4260,7 @@ struct Tensor FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { verifier.VerifyTable(sparsity()) && VerifyOffset(verifier, VT_SHAPE_SIGNATURE) && verifier.VerifyVector(shape_signature()) && + VerifyField(verifier, VT_HAS_RANK) && verifier.EndTable(); } TensorT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -4015,6 +4269,7 @@ struct Tensor FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct TensorBuilder { + typedef Tensor Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_shape(flatbuffers::Offset> shape) { @@ -4041,6 +4296,9 @@ struct TensorBuilder { void add_shape_signature(flatbuffers::Offset> shape_signature) { fbb_.AddOffset(Tensor::VT_SHAPE_SIGNATURE, shape_signature); } + void add_has_rank(bool has_rank) { + fbb_.AddElement(Tensor::VT_HAS_RANK, static_cast(has_rank), 0); + } explicit TensorBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -4062,7 +4320,8 @@ inline flatbuffers::Offset CreateTensor( flatbuffers::Offset quantization = 0, bool is_variable = false, flatbuffers::Offset sparsity = 0, - flatbuffers::Offset> shape_signature = 0) { + flatbuffers::Offset> shape_signature = 0, + bool has_rank = false) { TensorBuilder builder_(_fbb); builder_.add_shape_signature(shape_signature); builder_.add_sparsity(sparsity); @@ -4070,6 +4329,7 @@ inline flatbuffers::Offset CreateTensor( builder_.add_name(name); builder_.add_buffer(buffer); builder_.add_shape(shape); + builder_.add_has_rank(has_rank); builder_.add_is_variable(is_variable); builder_.add_type(type); return builder_.Finish(); @@ -4084,7 +4344,8 @@ inline flatbuffers::Offset CreateTensorDirect( flatbuffers::Offset quantization = 0, bool is_variable = false, flatbuffers::Offset sparsity = 0, - const std::vector *shape_signature = nullptr) { + const std::vector *shape_signature = nullptr, + bool has_rank = false) { auto shape__ = shape ? _fbb.CreateVector(*shape) : 0; auto name__ = name ? _fbb.CreateString(name) : 0; auto shape_signature__ = shape_signature ? _fbb.CreateVector(*shape_signature) : 0; @@ -4097,7 +4358,8 @@ inline flatbuffers::Offset CreateTensorDirect( quantization, is_variable, sparsity, - shape_signature__); + shape_signature__, + has_rank); } flatbuffers::Offset CreateTensor(flatbuffers::FlatBufferBuilder &_fbb, const TensorT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); @@ -4122,6 +4384,7 @@ struct Conv2DOptionsT : public flatbuffers::NativeTable { struct Conv2DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef Conv2DOptionsT NativeTableType; + typedef Conv2DOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_PADDING = 4, VT_STRIDE_W = 6, @@ -4164,6 +4427,7 @@ struct Conv2DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct Conv2DOptionsBuilder { + typedef Conv2DOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_padding(tflite::Padding padding) { @@ -4240,6 +4504,7 @@ struct Conv3DOptionsT : public flatbuffers::NativeTable { struct Conv3DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef Conv3DOptionsT NativeTableType; + typedef Conv3DOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_PADDING = 4, VT_STRIDE_D = 6, @@ -4292,6 +4557,7 @@ struct Conv3DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct Conv3DOptionsBuilder { + typedef Conv3DOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_padding(tflite::Padding padding) { @@ -4374,6 +4640,7 @@ struct Pool2DOptionsT : public flatbuffers::NativeTable { struct Pool2DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef Pool2DOptionsT NativeTableType; + typedef Pool2DOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_PADDING = 4, VT_STRIDE_W = 6, @@ -4416,6 +4683,7 @@ struct Pool2DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct Pool2DOptionsBuilder { + typedef Pool2DOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_padding(tflite::Padding padding) { @@ -4490,6 +4758,7 @@ struct DepthwiseConv2DOptionsT : public flatbuffers::NativeTable { struct DepthwiseConv2DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef DepthwiseConv2DOptionsT NativeTableType; + typedef DepthwiseConv2DOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_PADDING = 4, VT_STRIDE_W = 6, @@ -4537,6 +4806,7 @@ struct DepthwiseConv2DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab }; struct DepthwiseConv2DOptionsBuilder { + typedef DepthwiseConv2DOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_padding(tflite::Padding padding) { @@ -4606,6 +4876,7 @@ struct ConcatEmbeddingsOptionsT : public flatbuffers::NativeTable { struct ConcatEmbeddingsOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ConcatEmbeddingsOptionsT NativeTableType; + typedef ConcatEmbeddingsOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_NUM_CHANNELS = 4, VT_NUM_COLUMNS_PER_CHANNEL = 6, @@ -4635,6 +4906,7 @@ struct ConcatEmbeddingsOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Ta }; struct ConcatEmbeddingsOptionsBuilder { + typedef ConcatEmbeddingsOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_num_channels(int32_t num_channels) { @@ -4696,6 +4968,7 @@ struct LSHProjectionOptionsT : public flatbuffers::NativeTable { struct LSHProjectionOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LSHProjectionOptionsT NativeTableType; + typedef LSHProjectionOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_TYPE = 4 }; @@ -4713,6 +4986,7 @@ struct LSHProjectionOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct LSHProjectionOptionsBuilder { + typedef LSHProjectionOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_type(tflite::LSHProjectionType type) { @@ -4754,6 +5028,7 @@ struct SVDFOptionsT : public flatbuffers::NativeTable { struct SVDFOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SVDFOptionsT NativeTableType; + typedef SVDFOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_RANK = 4, VT_FUSED_ACTIVATION_FUNCTION = 6, @@ -4781,6 +5056,7 @@ struct SVDFOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SVDFOptionsBuilder { + typedef SVDFOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_rank(int32_t rank) { @@ -4830,6 +5106,7 @@ struct RNNOptionsT : public flatbuffers::NativeTable { struct RNNOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef RNNOptionsT NativeTableType; + typedef RNNOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4, VT_ASYMMETRIC_QUANTIZE_INPUTS = 6 @@ -4852,6 +5129,7 @@ struct RNNOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct RNNOptionsBuilder { + typedef RNNOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -4898,6 +5176,7 @@ struct SequenceRNNOptionsT : public flatbuffers::NativeTable { struct SequenceRNNOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SequenceRNNOptionsT NativeTableType; + typedef SequenceRNNOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_TIME_MAJOR = 4, VT_FUSED_ACTIVATION_FUNCTION = 6, @@ -4925,6 +5204,7 @@ struct SequenceRNNOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SequenceRNNOptionsBuilder { + typedef SequenceRNNOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_time_major(bool time_major) { @@ -4978,6 +5258,7 @@ struct BidirectionalSequenceRNNOptionsT : public flatbuffers::NativeTable { struct BidirectionalSequenceRNNOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef BidirectionalSequenceRNNOptionsT NativeTableType; + typedef BidirectionalSequenceRNNOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_TIME_MAJOR = 4, VT_FUSED_ACTIVATION_FUNCTION = 6, @@ -5010,6 +5291,7 @@ struct BidirectionalSequenceRNNOptions FLATBUFFERS_FINAL_CLASS : private flatbuf }; struct BidirectionalSequenceRNNOptionsBuilder { + typedef BidirectionalSequenceRNNOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_time_major(bool time_major) { @@ -5068,6 +5350,7 @@ struct FullyConnectedOptionsT : public flatbuffers::NativeTable { struct FullyConnectedOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef FullyConnectedOptionsT NativeTableType; + typedef FullyConnectedOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4, VT_WEIGHTS_FORMAT = 6, @@ -5100,6 +5383,7 @@ struct FullyConnectedOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl }; struct FullyConnectedOptionsBuilder { + typedef FullyConnectedOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -5152,6 +5436,7 @@ struct SoftmaxOptionsT : public flatbuffers::NativeTable { struct SoftmaxOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SoftmaxOptionsT NativeTableType; + typedef SoftmaxOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_BETA = 4 }; @@ -5169,6 +5454,7 @@ struct SoftmaxOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SoftmaxOptionsBuilder { + typedef SoftmaxOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_beta(float beta) { @@ -5208,6 +5494,7 @@ struct ConcatenationOptionsT : public flatbuffers::NativeTable { struct ConcatenationOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ConcatenationOptionsT NativeTableType; + typedef ConcatenationOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_AXIS = 4, VT_FUSED_ACTIVATION_FUNCTION = 6 @@ -5230,6 +5517,7 @@ struct ConcatenationOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct ConcatenationOptionsBuilder { + typedef ConcatenationOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_axis(int32_t axis) { @@ -5274,6 +5562,7 @@ struct AddOptionsT : public flatbuffers::NativeTable { struct AddOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef AddOptionsT NativeTableType; + typedef AddOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4, VT_POT_SCALE_INT16 = 6 @@ -5296,6 +5585,7 @@ struct AddOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct AddOptionsBuilder { + typedef AddOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -5338,6 +5628,7 @@ struct MulOptionsT : public flatbuffers::NativeTable { struct MulOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef MulOptionsT NativeTableType; + typedef MulOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4 }; @@ -5355,6 +5646,7 @@ struct MulOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct MulOptionsBuilder { + typedef MulOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -5392,6 +5684,7 @@ struct L2NormOptionsT : public flatbuffers::NativeTable { struct L2NormOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef L2NormOptionsT NativeTableType; + typedef L2NormOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4 }; @@ -5409,6 +5702,7 @@ struct L2NormOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct L2NormOptionsBuilder { + typedef L2NormOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -5452,6 +5746,7 @@ struct LocalResponseNormalizationOptionsT : public flatbuffers::NativeTable { struct LocalResponseNormalizationOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LocalResponseNormalizationOptionsT NativeTableType; + typedef LocalResponseNormalizationOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_RADIUS = 4, VT_BIAS = 6, @@ -5484,6 +5779,7 @@ struct LocalResponseNormalizationOptions FLATBUFFERS_FINAL_CLASS : private flatb }; struct LocalResponseNormalizationOptionsBuilder { + typedef LocalResponseNormalizationOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_radius(int32_t radius) { @@ -5544,6 +5840,7 @@ struct LSTMOptionsT : public flatbuffers::NativeTable { struct LSTMOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LSTMOptionsT NativeTableType; + typedef LSTMOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4, VT_CELL_CLIP = 6, @@ -5581,6 +5878,7 @@ struct LSTMOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct LSTMOptionsBuilder { + typedef LSTMOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -5646,6 +5944,7 @@ struct UnidirectionalSequenceLSTMOptionsT : public flatbuffers::NativeTable { struct UnidirectionalSequenceLSTMOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef UnidirectionalSequenceLSTMOptionsT NativeTableType; + typedef UnidirectionalSequenceLSTMOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4, VT_CELL_CLIP = 6, @@ -5683,6 +5982,7 @@ struct UnidirectionalSequenceLSTMOptions FLATBUFFERS_FINAL_CLASS : private flatb }; struct UnidirectionalSequenceLSTMOptionsBuilder { + typedef UnidirectionalSequenceLSTMOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -5750,6 +6050,7 @@ struct BidirectionalSequenceLSTMOptionsT : public flatbuffers::NativeTable { struct BidirectionalSequenceLSTMOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef BidirectionalSequenceLSTMOptionsT NativeTableType; + typedef BidirectionalSequenceLSTMOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4, VT_CELL_CLIP = 6, @@ -5792,6 +6093,7 @@ struct BidirectionalSequenceLSTMOptions FLATBUFFERS_FINAL_CLASS : private flatbu }; struct BidirectionalSequenceLSTMOptionsBuilder { + typedef BidirectionalSequenceLSTMOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -5856,6 +6158,7 @@ struct ResizeBilinearOptionsT : public flatbuffers::NativeTable { struct ResizeBilinearOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ResizeBilinearOptionsT NativeTableType; + typedef ResizeBilinearOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_ALIGN_CORNERS = 8, VT_HALF_PIXEL_CENTERS = 10 @@ -5878,6 +6181,7 @@ struct ResizeBilinearOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl }; struct ResizeBilinearOptionsBuilder { + typedef ResizeBilinearOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_align_corners(bool align_corners) { @@ -5922,6 +6226,7 @@ struct ResizeNearestNeighborOptionsT : public flatbuffers::NativeTable { struct ResizeNearestNeighborOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ResizeNearestNeighborOptionsT NativeTableType; + typedef ResizeNearestNeighborOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_ALIGN_CORNERS = 4, VT_HALF_PIXEL_CENTERS = 6 @@ -5944,6 +6249,7 @@ struct ResizeNearestNeighborOptions FLATBUFFERS_FINAL_CLASS : private flatbuffer }; struct ResizeNearestNeighborOptionsBuilder { + typedef ResizeNearestNeighborOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_align_corners(bool align_corners) { @@ -5986,6 +6292,7 @@ struct CallOptionsT : public flatbuffers::NativeTable { struct CallOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef CallOptionsT NativeTableType; + typedef CallOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_SUBGRAPH = 4 }; @@ -6003,6 +6310,7 @@ struct CallOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct CallOptionsBuilder { + typedef CallOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_subgraph(uint32_t subgraph) { @@ -6038,6 +6346,7 @@ struct PadOptionsT : public flatbuffers::NativeTable { struct PadOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef PadOptionsT NativeTableType; + typedef PadOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -6048,6 +6357,7 @@ struct PadOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct PadOptionsBuilder { + typedef PadOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit PadOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -6078,6 +6388,7 @@ struct PadV2OptionsT : public flatbuffers::NativeTable { struct PadV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef PadV2OptionsT NativeTableType; + typedef PadV2OptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -6088,6 +6399,7 @@ struct PadV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct PadV2OptionsBuilder { + typedef PadV2Options Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit PadV2OptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -6119,6 +6431,7 @@ struct ReshapeOptionsT : public flatbuffers::NativeTable { struct ReshapeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ReshapeOptionsT NativeTableType; + typedef ReshapeOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_NEW_SHAPE = 4 }; @@ -6137,6 +6450,7 @@ struct ReshapeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ReshapeOptionsBuilder { + typedef ReshapeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_new_shape(flatbuffers::Offset> new_shape) { @@ -6181,6 +6495,7 @@ struct SpaceToBatchNDOptionsT : public flatbuffers::NativeTable { struct SpaceToBatchNDOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SpaceToBatchNDOptionsT NativeTableType; + typedef SpaceToBatchNDOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -6191,6 +6506,7 @@ struct SpaceToBatchNDOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl }; struct SpaceToBatchNDOptionsBuilder { + typedef SpaceToBatchNDOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit SpaceToBatchNDOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -6221,6 +6537,7 @@ struct BatchToSpaceNDOptionsT : public flatbuffers::NativeTable { struct BatchToSpaceNDOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef BatchToSpaceNDOptionsT NativeTableType; + typedef BatchToSpaceNDOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -6231,6 +6548,7 @@ struct BatchToSpaceNDOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl }; struct BatchToSpaceNDOptionsBuilder { + typedef BatchToSpaceNDOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit BatchToSpaceNDOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -6267,6 +6585,7 @@ struct SkipGramOptionsT : public flatbuffers::NativeTable { struct SkipGramOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SkipGramOptionsT NativeTableType; + typedef SkipGramOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_NGRAM_SIZE = 4, VT_MAX_SKIP_SIZE = 6, @@ -6294,6 +6613,7 @@ struct SkipGramOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SkipGramOptionsBuilder { + typedef SkipGramOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_ngram_size(int32_t ngram_size) { @@ -6341,6 +6661,7 @@ struct SpaceToDepthOptionsT : public flatbuffers::NativeTable { struct SpaceToDepthOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SpaceToDepthOptionsT NativeTableType; + typedef SpaceToDepthOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_BLOCK_SIZE = 4 }; @@ -6358,6 +6679,7 @@ struct SpaceToDepthOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct SpaceToDepthOptionsBuilder { + typedef SpaceToDepthOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_block_size(int32_t block_size) { @@ -6395,6 +6717,7 @@ struct DepthToSpaceOptionsT : public flatbuffers::NativeTable { struct DepthToSpaceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef DepthToSpaceOptionsT NativeTableType; + typedef DepthToSpaceOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_BLOCK_SIZE = 4 }; @@ -6412,6 +6735,7 @@ struct DepthToSpaceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct DepthToSpaceOptionsBuilder { + typedef DepthToSpaceOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_block_size(int32_t block_size) { @@ -6451,6 +6775,7 @@ struct SubOptionsT : public flatbuffers::NativeTable { struct SubOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SubOptionsT NativeTableType; + typedef SubOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4, VT_POT_SCALE_INT16 = 6 @@ -6473,6 +6798,7 @@ struct SubOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SubOptionsBuilder { + typedef SubOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -6515,6 +6841,7 @@ struct DivOptionsT : public flatbuffers::NativeTable { struct DivOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef DivOptionsT NativeTableType; + typedef DivOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_FUSED_ACTIVATION_FUNCTION = 4 }; @@ -6532,6 +6859,7 @@ struct DivOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct DivOptionsBuilder { + typedef DivOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_fused_activation_function(tflite::ActivationFunctionType fused_activation_function) { @@ -6567,6 +6895,7 @@ struct TopKV2OptionsT : public flatbuffers::NativeTable { struct TopKV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef TopKV2OptionsT NativeTableType; + typedef TopKV2OptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -6577,6 +6906,7 @@ struct TopKV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct TopKV2OptionsBuilder { + typedef TopKV2Options Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit TopKV2OptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -6609,6 +6939,7 @@ struct EmbeddingLookupSparseOptionsT : public flatbuffers::NativeTable { struct EmbeddingLookupSparseOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef EmbeddingLookupSparseOptionsT NativeTableType; + typedef EmbeddingLookupSparseOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_COMBINER = 4 }; @@ -6626,6 +6957,7 @@ struct EmbeddingLookupSparseOptions FLATBUFFERS_FINAL_CLASS : private flatbuffer }; struct EmbeddingLookupSparseOptionsBuilder { + typedef EmbeddingLookupSparseOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_combiner(tflite::CombinerType combiner) { @@ -6665,6 +6997,7 @@ struct GatherOptionsT : public flatbuffers::NativeTable { struct GatherOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef GatherOptionsT NativeTableType; + typedef GatherOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_AXIS = 4, VT_BATCH_DIMS = 6 @@ -6687,6 +7020,7 @@ struct GatherOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct GatherOptionsBuilder { + typedef GatherOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_axis(int32_t axis) { @@ -6727,6 +7061,7 @@ struct TransposeOptionsT : public flatbuffers::NativeTable { struct TransposeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef TransposeOptionsT NativeTableType; + typedef TransposeOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -6737,6 +7072,7 @@ struct TransposeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct TransposeOptionsBuilder { + typedef TransposeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit TransposeOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -6767,6 +7103,7 @@ struct ExpOptionsT : public flatbuffers::NativeTable { struct ExpOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ExpOptionsT NativeTableType; + typedef ExpOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -6777,6 +7114,7 @@ struct ExpOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ExpOptionsBuilder { + typedef ExpOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit ExpOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -6807,6 +7145,7 @@ struct CosOptionsT : public flatbuffers::NativeTable { struct CosOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef CosOptionsT NativeTableType; + typedef CosOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -6817,6 +7156,7 @@ struct CosOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct CosOptionsBuilder { + typedef CosOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit CosOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -6849,6 +7189,7 @@ struct ReducerOptionsT : public flatbuffers::NativeTable { struct ReducerOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ReducerOptionsT NativeTableType; + typedef ReducerOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_KEEP_DIMS = 4 }; @@ -6866,6 +7207,7 @@ struct ReducerOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ReducerOptionsBuilder { + typedef ReducerOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_keep_dims(bool keep_dims) { @@ -6902,6 +7244,7 @@ struct SqueezeOptionsT : public flatbuffers::NativeTable { struct SqueezeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SqueezeOptionsT NativeTableType; + typedef SqueezeOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_SQUEEZE_DIMS = 4 }; @@ -6920,6 +7263,7 @@ struct SqueezeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SqueezeOptionsBuilder { + typedef SqueezeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_squeeze_dims(flatbuffers::Offset> squeeze_dims) { @@ -6966,6 +7310,7 @@ struct SplitOptionsT : public flatbuffers::NativeTable { struct SplitOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SplitOptionsT NativeTableType; + typedef SplitOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_NUM_SPLITS = 4 }; @@ -6983,6 +7328,7 @@ struct SplitOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SplitOptionsBuilder { + typedef SplitOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_num_splits(int32_t num_splits) { @@ -7020,6 +7366,7 @@ struct SplitVOptionsT : public flatbuffers::NativeTable { struct SplitVOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SplitVOptionsT NativeTableType; + typedef SplitVOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_NUM_SPLITS = 4 }; @@ -7037,6 +7384,7 @@ struct SplitVOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SplitVOptionsBuilder { + typedef SplitVOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_num_splits(int32_t num_splits) { @@ -7082,6 +7430,7 @@ struct StridedSliceOptionsT : public flatbuffers::NativeTable { struct StridedSliceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef StridedSliceOptionsT NativeTableType; + typedef StridedSliceOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_BEGIN_MASK = 4, VT_END_MASK = 6, @@ -7119,6 +7468,7 @@ struct StridedSliceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct StridedSliceOptionsBuilder { + typedef StridedSliceOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_begin_mask(int32_t begin_mask) { @@ -7174,6 +7524,7 @@ struct LogSoftmaxOptionsT : public flatbuffers::NativeTable { struct LogSoftmaxOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LogSoftmaxOptionsT NativeTableType; + typedef LogSoftmaxOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7184,6 +7535,7 @@ struct LogSoftmaxOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct LogSoftmaxOptionsBuilder { + typedef LogSoftmaxOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit LogSoftmaxOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7218,6 +7570,7 @@ struct CastOptionsT : public flatbuffers::NativeTable { struct CastOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef CastOptionsT NativeTableType; + typedef CastOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_IN_DATA_TYPE = 4, VT_OUT_DATA_TYPE = 6 @@ -7240,6 +7593,7 @@ struct CastOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct CastOptionsBuilder { + typedef CastOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_in_data_type(tflite::TensorType in_data_type) { @@ -7280,6 +7634,7 @@ struct DequantizeOptionsT : public flatbuffers::NativeTable { struct DequantizeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef DequantizeOptionsT NativeTableType; + typedef DequantizeOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7290,6 +7645,7 @@ struct DequantizeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct DequantizeOptionsBuilder { + typedef DequantizeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit DequantizeOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7320,6 +7676,7 @@ struct MaximumMinimumOptionsT : public flatbuffers::NativeTable { struct MaximumMinimumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef MaximumMinimumOptionsT NativeTableType; + typedef MaximumMinimumOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7330,6 +7687,7 @@ struct MaximumMinimumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl }; struct MaximumMinimumOptionsBuilder { + typedef MaximumMinimumOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit MaximumMinimumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7360,6 +7718,7 @@ struct TileOptionsT : public flatbuffers::NativeTable { struct TileOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef TileOptionsT NativeTableType; + typedef TileOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7370,6 +7729,7 @@ struct TileOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct TileOptionsBuilder { + typedef TileOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit TileOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7402,6 +7762,7 @@ struct ArgMaxOptionsT : public flatbuffers::NativeTable { struct ArgMaxOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ArgMaxOptionsT NativeTableType; + typedef ArgMaxOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_OUTPUT_TYPE = 4 }; @@ -7419,6 +7780,7 @@ struct ArgMaxOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ArgMaxOptionsBuilder { + typedef ArgMaxOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_output_type(tflite::TensorType output_type) { @@ -7456,6 +7818,7 @@ struct ArgMinOptionsT : public flatbuffers::NativeTable { struct ArgMinOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ArgMinOptionsT NativeTableType; + typedef ArgMinOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_OUTPUT_TYPE = 4 }; @@ -7473,6 +7836,7 @@ struct ArgMinOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ArgMinOptionsBuilder { + typedef ArgMinOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_output_type(tflite::TensorType output_type) { @@ -7508,6 +7872,7 @@ struct GreaterOptionsT : public flatbuffers::NativeTable { struct GreaterOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef GreaterOptionsT NativeTableType; + typedef GreaterOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7518,6 +7883,7 @@ struct GreaterOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct GreaterOptionsBuilder { + typedef GreaterOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit GreaterOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7548,6 +7914,7 @@ struct GreaterEqualOptionsT : public flatbuffers::NativeTable { struct GreaterEqualOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef GreaterEqualOptionsT NativeTableType; + typedef GreaterEqualOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7558,6 +7925,7 @@ struct GreaterEqualOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct GreaterEqualOptionsBuilder { + typedef GreaterEqualOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit GreaterEqualOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7588,6 +7956,7 @@ struct LessOptionsT : public flatbuffers::NativeTable { struct LessOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LessOptionsT NativeTableType; + typedef LessOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7598,6 +7967,7 @@ struct LessOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct LessOptionsBuilder { + typedef LessOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit LessOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7628,6 +7998,7 @@ struct LessEqualOptionsT : public flatbuffers::NativeTable { struct LessEqualOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LessEqualOptionsT NativeTableType; + typedef LessEqualOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7638,6 +8009,7 @@ struct LessEqualOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct LessEqualOptionsBuilder { + typedef LessEqualOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit LessEqualOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7668,6 +8040,7 @@ struct NegOptionsT : public flatbuffers::NativeTable { struct NegOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef NegOptionsT NativeTableType; + typedef NegOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7678,6 +8051,7 @@ struct NegOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct NegOptionsBuilder { + typedef NegOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit NegOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7708,6 +8082,7 @@ struct SelectOptionsT : public flatbuffers::NativeTable { struct SelectOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SelectOptionsT NativeTableType; + typedef SelectOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7718,6 +8093,7 @@ struct SelectOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SelectOptionsBuilder { + typedef SelectOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit SelectOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7748,6 +8124,7 @@ struct SliceOptionsT : public flatbuffers::NativeTable { struct SliceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SliceOptionsT NativeTableType; + typedef SliceOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7758,6 +8135,7 @@ struct SliceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SliceOptionsBuilder { + typedef SliceOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit SliceOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7794,6 +8172,7 @@ struct TransposeConvOptionsT : public flatbuffers::NativeTable { struct TransposeConvOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef TransposeConvOptionsT NativeTableType; + typedef TransposeConvOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_PADDING = 4, VT_STRIDE_W = 6, @@ -7821,6 +8200,7 @@ struct TransposeConvOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct TransposeConvOptionsBuilder { + typedef TransposeConvOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_padding(tflite::Padding padding) { @@ -7866,6 +8246,7 @@ struct ExpandDimsOptionsT : public flatbuffers::NativeTable { struct ExpandDimsOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ExpandDimsOptionsT NativeTableType; + typedef ExpandDimsOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7876,6 +8257,7 @@ struct ExpandDimsOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ExpandDimsOptionsBuilder { + typedef ExpandDimsOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit ExpandDimsOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -7908,6 +8290,7 @@ struct SparseToDenseOptionsT : public flatbuffers::NativeTable { struct SparseToDenseOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SparseToDenseOptionsT NativeTableType; + typedef SparseToDenseOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_VALIDATE_INDICES = 4 }; @@ -7925,6 +8308,7 @@ struct SparseToDenseOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct SparseToDenseOptionsBuilder { + typedef SparseToDenseOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_validate_indices(bool validate_indices) { @@ -7960,6 +8344,7 @@ struct EqualOptionsT : public flatbuffers::NativeTable { struct EqualOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef EqualOptionsT NativeTableType; + typedef EqualOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -7970,6 +8355,7 @@ struct EqualOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct EqualOptionsBuilder { + typedef EqualOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit EqualOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8000,6 +8386,7 @@ struct NotEqualOptionsT : public flatbuffers::NativeTable { struct NotEqualOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef NotEqualOptionsT NativeTableType; + typedef NotEqualOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8010,6 +8397,7 @@ struct NotEqualOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct NotEqualOptionsBuilder { + typedef NotEqualOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit NotEqualOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8042,6 +8430,7 @@ struct ShapeOptionsT : public flatbuffers::NativeTable { struct ShapeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ShapeOptionsT NativeTableType; + typedef ShapeOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_OUT_TYPE = 4 }; @@ -8059,6 +8448,7 @@ struct ShapeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ShapeOptionsBuilder { + typedef ShapeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_out_type(tflite::TensorType out_type) { @@ -8094,6 +8484,7 @@ struct RankOptionsT : public flatbuffers::NativeTable { struct RankOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef RankOptionsT NativeTableType; + typedef RankOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8104,6 +8495,7 @@ struct RankOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct RankOptionsBuilder { + typedef RankOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit RankOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8134,6 +8526,7 @@ struct PowOptionsT : public flatbuffers::NativeTable { struct PowOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef PowOptionsT NativeTableType; + typedef PowOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8144,6 +8537,7 @@ struct PowOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct PowOptionsBuilder { + typedef PowOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit PowOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8182,6 +8576,7 @@ struct FakeQuantOptionsT : public flatbuffers::NativeTable { struct FakeQuantOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef FakeQuantOptionsT NativeTableType; + typedef FakeQuantOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_MIN = 4, VT_MAX = 6, @@ -8214,6 +8609,7 @@ struct FakeQuantOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct FakeQuantOptionsBuilder { + typedef FakeQuantOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_min(float min) { @@ -8268,6 +8664,7 @@ struct PackOptionsT : public flatbuffers::NativeTable { struct PackOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef PackOptionsT NativeTableType; + typedef PackOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_VALUES_COUNT = 4, VT_AXIS = 6 @@ -8290,6 +8687,7 @@ struct PackOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct PackOptionsBuilder { + typedef PackOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_values_count(int32_t values_count) { @@ -8330,6 +8728,7 @@ struct LogicalOrOptionsT : public flatbuffers::NativeTable { struct LogicalOrOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LogicalOrOptionsT NativeTableType; + typedef LogicalOrOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8340,6 +8739,7 @@ struct LogicalOrOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct LogicalOrOptionsBuilder { + typedef LogicalOrOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit LogicalOrOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8372,6 +8772,7 @@ struct OneHotOptionsT : public flatbuffers::NativeTable { struct OneHotOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef OneHotOptionsT NativeTableType; + typedef OneHotOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_AXIS = 4 }; @@ -8389,6 +8790,7 @@ struct OneHotOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct OneHotOptionsBuilder { + typedef OneHotOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_axis(int32_t axis) { @@ -8424,6 +8826,7 @@ struct AbsOptionsT : public flatbuffers::NativeTable { struct AbsOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef AbsOptionsT NativeTableType; + typedef AbsOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8434,6 +8837,7 @@ struct AbsOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct AbsOptionsBuilder { + typedef AbsOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit AbsOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8464,6 +8868,7 @@ struct HardSwishOptionsT : public flatbuffers::NativeTable { struct HardSwishOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef HardSwishOptionsT NativeTableType; + typedef HardSwishOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8474,6 +8879,7 @@ struct HardSwishOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct HardSwishOptionsBuilder { + typedef HardSwishOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit HardSwishOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8504,6 +8910,7 @@ struct LogicalAndOptionsT : public flatbuffers::NativeTable { struct LogicalAndOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LogicalAndOptionsT NativeTableType; + typedef LogicalAndOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8514,6 +8921,7 @@ struct LogicalAndOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct LogicalAndOptionsBuilder { + typedef LogicalAndOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit LogicalAndOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8544,6 +8952,7 @@ struct LogicalNotOptionsT : public flatbuffers::NativeTable { struct LogicalNotOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LogicalNotOptionsT NativeTableType; + typedef LogicalNotOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8554,6 +8963,7 @@ struct LogicalNotOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct LogicalNotOptionsBuilder { + typedef LogicalNotOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit LogicalNotOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8588,6 +8998,7 @@ struct UnpackOptionsT : public flatbuffers::NativeTable { struct UnpackOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef UnpackOptionsT NativeTableType; + typedef UnpackOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_NUM = 4, VT_AXIS = 6 @@ -8610,6 +9021,7 @@ struct UnpackOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct UnpackOptionsBuilder { + typedef UnpackOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_num(int32_t num) { @@ -8650,6 +9062,7 @@ struct FloorDivOptionsT : public flatbuffers::NativeTable { struct FloorDivOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef FloorDivOptionsT NativeTableType; + typedef FloorDivOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8660,6 +9073,7 @@ struct FloorDivOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct FloorDivOptionsBuilder { + typedef FloorDivOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit FloorDivOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8690,6 +9104,7 @@ struct SquareOptionsT : public flatbuffers::NativeTable { struct SquareOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SquareOptionsT NativeTableType; + typedef SquareOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8700,6 +9115,7 @@ struct SquareOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SquareOptionsBuilder { + typedef SquareOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit SquareOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8730,6 +9146,7 @@ struct ZerosLikeOptionsT : public flatbuffers::NativeTable { struct ZerosLikeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ZerosLikeOptionsT NativeTableType; + typedef ZerosLikeOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8740,6 +9157,7 @@ struct ZerosLikeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ZerosLikeOptionsBuilder { + typedef ZerosLikeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit ZerosLikeOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8770,6 +9188,7 @@ struct FillOptionsT : public flatbuffers::NativeTable { struct FillOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef FillOptionsT NativeTableType; + typedef FillOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8780,6 +9199,7 @@ struct FillOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct FillOptionsBuilder { + typedef FillOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit FillOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8810,6 +9230,7 @@ struct FloorModOptionsT : public flatbuffers::NativeTable { struct FloorModOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef FloorModOptionsT NativeTableType; + typedef FloorModOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8820,6 +9241,7 @@ struct FloorModOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct FloorModOptionsBuilder { + typedef FloorModOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit FloorModOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8850,6 +9272,7 @@ struct RangeOptionsT : public flatbuffers::NativeTable { struct RangeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef RangeOptionsT NativeTableType; + typedef RangeOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8860,6 +9283,7 @@ struct RangeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct RangeOptionsBuilder { + typedef RangeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit RangeOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8892,6 +9316,7 @@ struct LeakyReluOptionsT : public flatbuffers::NativeTable { struct LeakyReluOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef LeakyReluOptionsT NativeTableType; + typedef LeakyReluOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_ALPHA = 4 }; @@ -8909,6 +9334,7 @@ struct LeakyReluOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct LeakyReluOptionsBuilder { + typedef LeakyReluOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_alpha(float alpha) { @@ -8944,6 +9370,7 @@ struct SquaredDifferenceOptionsT : public flatbuffers::NativeTable { struct SquaredDifferenceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SquaredDifferenceOptionsT NativeTableType; + typedef SquaredDifferenceOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -8954,6 +9381,7 @@ struct SquaredDifferenceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::T }; struct SquaredDifferenceOptionsBuilder { + typedef SquaredDifferenceOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit SquaredDifferenceOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -8986,6 +9414,7 @@ struct MirrorPadOptionsT : public flatbuffers::NativeTable { struct MirrorPadOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef MirrorPadOptionsT NativeTableType; + typedef MirrorPadOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_MODE = 4 }; @@ -9003,6 +9432,7 @@ struct MirrorPadOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct MirrorPadOptionsBuilder { + typedef MirrorPadOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_mode(tflite::MirrorPadMode mode) { @@ -9040,6 +9470,7 @@ struct UniqueOptionsT : public flatbuffers::NativeTable { struct UniqueOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef UniqueOptionsT NativeTableType; + typedef UniqueOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_IDX_OUT_TYPE = 4 }; @@ -9057,6 +9488,7 @@ struct UniqueOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct UniqueOptionsBuilder { + typedef UniqueOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_idx_out_type(tflite::TensorType idx_out_type) { @@ -9092,6 +9524,7 @@ struct ReverseV2OptionsT : public flatbuffers::NativeTable { struct ReverseV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ReverseV2OptionsT NativeTableType; + typedef ReverseV2OptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9102,6 +9535,7 @@ struct ReverseV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ReverseV2OptionsBuilder { + typedef ReverseV2Options Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit ReverseV2OptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9132,6 +9566,7 @@ struct AddNOptionsT : public flatbuffers::NativeTable { struct AddNOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef AddNOptionsT NativeTableType; + typedef AddNOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9142,6 +9577,7 @@ struct AddNOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct AddNOptionsBuilder { + typedef AddNOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit AddNOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9172,6 +9608,7 @@ struct GatherNdOptionsT : public flatbuffers::NativeTable { struct GatherNdOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef GatherNdOptionsT NativeTableType; + typedef GatherNdOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9182,6 +9619,7 @@ struct GatherNdOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct GatherNdOptionsBuilder { + typedef GatherNdOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit GatherNdOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9212,6 +9650,7 @@ struct WhereOptionsT : public flatbuffers::NativeTable { struct WhereOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef WhereOptionsT NativeTableType; + typedef WhereOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9222,6 +9661,7 @@ struct WhereOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct WhereOptionsBuilder { + typedef WhereOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit WhereOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9256,6 +9696,7 @@ struct ReverseSequenceOptionsT : public flatbuffers::NativeTable { struct ReverseSequenceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ReverseSequenceOptionsT NativeTableType; + typedef ReverseSequenceOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_SEQ_DIM = 4, VT_BATCH_DIM = 6 @@ -9278,6 +9719,7 @@ struct ReverseSequenceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab }; struct ReverseSequenceOptionsBuilder { + typedef ReverseSequenceOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_seq_dim(int32_t seq_dim) { @@ -9318,6 +9760,7 @@ struct MatrixDiagOptionsT : public flatbuffers::NativeTable { struct MatrixDiagOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef MatrixDiagOptionsT NativeTableType; + typedef MatrixDiagOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9328,6 +9771,7 @@ struct MatrixDiagOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct MatrixDiagOptionsBuilder { + typedef MatrixDiagOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit MatrixDiagOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9358,6 +9802,7 @@ struct QuantizeOptionsT : public flatbuffers::NativeTable { struct QuantizeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef QuantizeOptionsT NativeTableType; + typedef QuantizeOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9368,6 +9813,7 @@ struct QuantizeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct QuantizeOptionsBuilder { + typedef QuantizeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit QuantizeOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9398,6 +9844,7 @@ struct MatrixSetDiagOptionsT : public flatbuffers::NativeTable { struct MatrixSetDiagOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef MatrixSetDiagOptionsT NativeTableType; + typedef MatrixSetDiagOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9408,6 +9855,7 @@ struct MatrixSetDiagOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct MatrixSetDiagOptionsBuilder { + typedef MatrixSetDiagOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit MatrixSetDiagOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9442,6 +9890,7 @@ struct IfOptionsT : public flatbuffers::NativeTable { struct IfOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef IfOptionsT NativeTableType; + typedef IfOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_THEN_SUBGRAPH_INDEX = 4, VT_ELSE_SUBGRAPH_INDEX = 6 @@ -9464,6 +9913,7 @@ struct IfOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct IfOptionsBuilder { + typedef IfOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_then_subgraph_index(int32_t then_subgraph_index) { @@ -9506,6 +9956,7 @@ struct CallOnceOptionsT : public flatbuffers::NativeTable { struct CallOnceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef CallOnceOptionsT NativeTableType; + typedef CallOnceOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_INIT_SUBGRAPH_INDEX = 4 }; @@ -9523,6 +9974,7 @@ struct CallOnceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct CallOnceOptionsBuilder { + typedef CallOnceOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_init_subgraph_index(int32_t init_subgraph_index) { @@ -9562,6 +10014,7 @@ struct WhileOptionsT : public flatbuffers::NativeTable { struct WhileOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef WhileOptionsT NativeTableType; + typedef WhileOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_COND_SUBGRAPH_INDEX = 4, VT_BODY_SUBGRAPH_INDEX = 6 @@ -9584,6 +10037,7 @@ struct WhileOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct WhileOptionsBuilder { + typedef WhileOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_cond_subgraph_index(int32_t cond_subgraph_index) { @@ -9624,6 +10078,7 @@ struct NonMaxSuppressionV4OptionsT : public flatbuffers::NativeTable { struct NonMaxSuppressionV4Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef NonMaxSuppressionV4OptionsT NativeTableType; + typedef NonMaxSuppressionV4OptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9634,6 +10089,7 @@ struct NonMaxSuppressionV4Options FLATBUFFERS_FINAL_CLASS : private flatbuffers: }; struct NonMaxSuppressionV4OptionsBuilder { + typedef NonMaxSuppressionV4Options Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit NonMaxSuppressionV4OptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9664,6 +10120,7 @@ struct NonMaxSuppressionV5OptionsT : public flatbuffers::NativeTable { struct NonMaxSuppressionV5Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef NonMaxSuppressionV5OptionsT NativeTableType; + typedef NonMaxSuppressionV5OptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9674,6 +10131,7 @@ struct NonMaxSuppressionV5Options FLATBUFFERS_FINAL_CLASS : private flatbuffers: }; struct NonMaxSuppressionV5OptionsBuilder { + typedef NonMaxSuppressionV5Options Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit NonMaxSuppressionV5OptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9704,6 +10162,7 @@ struct ScatterNdOptionsT : public flatbuffers::NativeTable { struct ScatterNdOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ScatterNdOptionsT NativeTableType; + typedef ScatterNdOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9714,6 +10173,7 @@ struct ScatterNdOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ScatterNdOptionsBuilder { + typedef ScatterNdOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit ScatterNdOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9744,6 +10204,7 @@ struct SelectV2OptionsT : public flatbuffers::NativeTable { struct SelectV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SelectV2OptionsT NativeTableType; + typedef SelectV2OptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9754,6 +10215,7 @@ struct SelectV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SelectV2OptionsBuilder { + typedef SelectV2Options Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit SelectV2OptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9784,6 +10246,7 @@ struct DensifyOptionsT : public flatbuffers::NativeTable { struct DensifyOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef DensifyOptionsT NativeTableType; + typedef DensifyOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9794,6 +10257,7 @@ struct DensifyOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct DensifyOptionsBuilder { + typedef DensifyOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit DensifyOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9824,6 +10288,7 @@ struct SegmentSumOptionsT : public flatbuffers::NativeTable { struct SegmentSumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SegmentSumOptionsT NativeTableType; + typedef SegmentSumOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -9834,6 +10299,7 @@ struct SegmentSumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SegmentSumOptionsBuilder { + typedef SegmentSumOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit SegmentSumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -9870,6 +10336,7 @@ struct BatchMatMulOptionsT : public flatbuffers::NativeTable { struct BatchMatMulOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef BatchMatMulOptionsT NativeTableType; + typedef BatchMatMulOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_ADJ_X = 4, VT_ADJ_Y = 6, @@ -9897,6 +10364,7 @@ struct BatchMatMulOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct BatchMatMulOptionsBuilder { + typedef BatchMatMulOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_adj_x(bool adj_x) { @@ -9946,6 +10414,7 @@ struct CumsumOptionsT : public flatbuffers::NativeTable { struct CumsumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef CumsumOptionsT NativeTableType; + typedef CumsumOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_EXCLUSIVE = 4, VT_REVERSE = 6 @@ -9968,6 +10437,7 @@ struct CumsumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct CumsumOptionsBuilder { + typedef CumsumOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_exclusive(bool exclusive) { @@ -10008,6 +10478,7 @@ struct BroadcastToOptionsT : public flatbuffers::NativeTable { struct BroadcastToOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef BroadcastToOptionsT NativeTableType; + typedef BroadcastToOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -10018,6 +10489,7 @@ struct BroadcastToOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct BroadcastToOptionsBuilder { + typedef BroadcastToOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit BroadcastToOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -10048,6 +10520,7 @@ struct Rfft2dOptionsT : public flatbuffers::NativeTable { struct Rfft2dOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef Rfft2dOptionsT NativeTableType; + typedef Rfft2dOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -10058,6 +10531,7 @@ struct Rfft2dOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct Rfft2dOptionsBuilder { + typedef Rfft2dOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit Rfft2dOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -10094,6 +10568,7 @@ struct HashtableOptionsT : public flatbuffers::NativeTable { struct HashtableOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef HashtableOptionsT NativeTableType; + typedef HashtableOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_TABLE_ID = 4, VT_KEY_DTYPE = 6, @@ -10121,6 +10596,7 @@ struct HashtableOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct HashtableOptionsBuilder { + typedef HashtableOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_table_id(int32_t table_id) { @@ -10166,6 +10642,7 @@ struct HashtableFindOptionsT : public flatbuffers::NativeTable { struct HashtableFindOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef HashtableFindOptionsT NativeTableType; + typedef HashtableFindOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -10176,6 +10653,7 @@ struct HashtableFindOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct HashtableFindOptionsBuilder { + typedef HashtableFindOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit HashtableFindOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -10206,6 +10684,7 @@ struct HashtableImportOptionsT : public flatbuffers::NativeTable { struct HashtableImportOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef HashtableImportOptionsT NativeTableType; + typedef HashtableImportOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -10216,6 +10695,7 @@ struct HashtableImportOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab }; struct HashtableImportOptionsBuilder { + typedef HashtableImportOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit HashtableImportOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -10246,6 +10726,7 @@ struct HashtableSizeOptionsT : public flatbuffers::NativeTable { struct HashtableSizeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef HashtableSizeOptionsT NativeTableType; + typedef HashtableSizeOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -10256,6 +10737,7 @@ struct HashtableSizeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct HashtableSizeOptionsBuilder { + typedef HashtableSizeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit HashtableSizeOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -10288,6 +10770,7 @@ struct VarHandleOptionsT : public flatbuffers::NativeTable { struct VarHandleOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef VarHandleOptionsT NativeTableType; + typedef VarHandleOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_CONTAINER = 4, VT_SHARED_NAME = 6 @@ -10312,6 +10795,7 @@ struct VarHandleOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct VarHandleOptionsBuilder { + typedef VarHandleOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_container(flatbuffers::Offset container) { @@ -10364,6 +10848,7 @@ struct ReadVariableOptionsT : public flatbuffers::NativeTable { struct ReadVariableOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ReadVariableOptionsT NativeTableType; + typedef ReadVariableOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -10374,6 +10859,7 @@ struct ReadVariableOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table }; struct ReadVariableOptionsBuilder { + typedef ReadVariableOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit ReadVariableOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -10404,6 +10890,7 @@ struct AssignVariableOptionsT : public flatbuffers::NativeTable { struct AssignVariableOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef AssignVariableOptionsT NativeTableType; + typedef AssignVariableOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -10414,6 +10901,7 @@ struct AssignVariableOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl }; struct AssignVariableOptionsBuilder { + typedef AssignVariableOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit AssignVariableOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -10448,6 +10936,7 @@ struct RandomOptionsT : public flatbuffers::NativeTable { struct RandomOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef RandomOptionsT NativeTableType; + typedef RandomOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_SEED = 4, VT_SEED2 = 6 @@ -10470,6 +10959,7 @@ struct RandomOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct RandomOptionsBuilder { + typedef RandomOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_seed(int64_t seed) { @@ -10511,6 +11001,7 @@ struct BucketizeOptionsT : public flatbuffers::NativeTable { struct BucketizeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef BucketizeOptionsT NativeTableType; + typedef BucketizeOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_BOUNDARIES = 4 }; @@ -10529,6 +11020,7 @@ struct BucketizeOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct BucketizeOptionsBuilder { + typedef BucketizeOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_boundaries(flatbuffers::Offset> boundaries) { @@ -10575,6 +11067,7 @@ struct GeluOptionsT : public flatbuffers::NativeTable { struct GeluOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef GeluOptionsT NativeTableType; + typedef GeluOptionsBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_APPROXIMATE = 4 }; @@ -10592,6 +11085,7 @@ struct GeluOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct GeluOptionsBuilder { + typedef GeluOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_approximate(bool approximate) { @@ -10627,6 +11121,7 @@ struct DynamicUpdateSliceOptionsT : public flatbuffers::NativeTable { struct DynamicUpdateSliceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef DynamicUpdateSliceOptionsT NativeTableType; + typedef DynamicUpdateSliceOptionsBuilder Builder; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); @@ -10637,6 +11132,7 @@ struct DynamicUpdateSliceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers:: }; struct DynamicUpdateSliceOptionsBuilder { + typedef DynamicUpdateSliceOptions Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; explicit DynamicUpdateSliceOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -10659,6 +11155,216 @@ inline flatbuffers::Offset CreateDynamicUpdateSliceOp flatbuffers::Offset CreateDynamicUpdateSliceOptions(flatbuffers::FlatBufferBuilder &_fbb, const DynamicUpdateSliceOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct UnsortedSegmentProdOptionsT : public flatbuffers::NativeTable { + typedef UnsortedSegmentProdOptions TableType; + UnsortedSegmentProdOptionsT() { + } +}; + +struct UnsortedSegmentProdOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef UnsortedSegmentProdOptionsT NativeTableType; + typedef UnsortedSegmentProdOptionsBuilder Builder; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + UnsortedSegmentProdOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(UnsortedSegmentProdOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentProdOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct UnsortedSegmentProdOptionsBuilder { + typedef UnsortedSegmentProdOptions Table; + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit UnsortedSegmentProdOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + UnsortedSegmentProdOptionsBuilder &operator=(const UnsortedSegmentProdOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateUnsortedSegmentProdOptions( + flatbuffers::FlatBufferBuilder &_fbb) { + UnsortedSegmentProdOptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateUnsortedSegmentProdOptions(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentProdOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct UnsortedSegmentMaxOptionsT : public flatbuffers::NativeTable { + typedef UnsortedSegmentMaxOptions TableType; + UnsortedSegmentMaxOptionsT() { + } +}; + +struct UnsortedSegmentMaxOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef UnsortedSegmentMaxOptionsT NativeTableType; + typedef UnsortedSegmentMaxOptionsBuilder Builder; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + UnsortedSegmentMaxOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(UnsortedSegmentMaxOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentMaxOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct UnsortedSegmentMaxOptionsBuilder { + typedef UnsortedSegmentMaxOptions Table; + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit UnsortedSegmentMaxOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + UnsortedSegmentMaxOptionsBuilder &operator=(const UnsortedSegmentMaxOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateUnsortedSegmentMaxOptions( + flatbuffers::FlatBufferBuilder &_fbb) { + UnsortedSegmentMaxOptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateUnsortedSegmentMaxOptions(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentMaxOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct UnsortedSegmentSumOptionsT : public flatbuffers::NativeTable { + typedef UnsortedSegmentSumOptions TableType; + UnsortedSegmentSumOptionsT() { + } +}; + +struct UnsortedSegmentSumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef UnsortedSegmentSumOptionsT NativeTableType; + typedef UnsortedSegmentSumOptionsBuilder Builder; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + UnsortedSegmentSumOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(UnsortedSegmentSumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentSumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct UnsortedSegmentSumOptionsBuilder { + typedef UnsortedSegmentSumOptions Table; + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit UnsortedSegmentSumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + UnsortedSegmentSumOptionsBuilder &operator=(const UnsortedSegmentSumOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateUnsortedSegmentSumOptions( + flatbuffers::FlatBufferBuilder &_fbb) { + UnsortedSegmentSumOptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateUnsortedSegmentSumOptions(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentSumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct ATan2OptionsT : public flatbuffers::NativeTable { + typedef ATan2Options TableType; + ATan2OptionsT() { + } +}; + +struct ATan2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef ATan2OptionsT NativeTableType; + typedef ATan2OptionsBuilder Builder; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + ATan2OptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(ATan2OptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const ATan2OptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct ATan2OptionsBuilder { + typedef ATan2Options Table; + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit ATan2OptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ATan2OptionsBuilder &operator=(const ATan2OptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateATan2Options( + flatbuffers::FlatBufferBuilder &_fbb) { + ATan2OptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateATan2Options(flatbuffers::FlatBufferBuilder &_fbb, const ATan2OptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct UnsortedSegmentMinOptionsT : public flatbuffers::NativeTable { + typedef UnsortedSegmentMinOptions TableType; + UnsortedSegmentMinOptionsT() { + } +}; + +struct UnsortedSegmentMinOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef UnsortedSegmentMinOptionsT NativeTableType; + typedef UnsortedSegmentMinOptionsBuilder Builder; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + UnsortedSegmentMinOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(UnsortedSegmentMinOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentMinOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct UnsortedSegmentMinOptionsBuilder { + typedef UnsortedSegmentMinOptions Table; + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit UnsortedSegmentMinOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + UnsortedSegmentMinOptionsBuilder &operator=(const UnsortedSegmentMinOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateUnsortedSegmentMinOptions( + flatbuffers::FlatBufferBuilder &_fbb) { + UnsortedSegmentMinOptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateUnsortedSegmentMinOptions(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentMinOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct OperatorCodeT : public flatbuffers::NativeTable { typedef OperatorCode TableType; int8_t deprecated_builtin_code; @@ -10674,6 +11380,7 @@ struct OperatorCodeT : public flatbuffers::NativeTable { struct OperatorCode FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef OperatorCodeT NativeTableType; + typedef OperatorCodeBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_DEPRECATED_BUILTIN_CODE = 4, VT_CUSTOM_CODE = 6, @@ -10707,6 +11414,7 @@ struct OperatorCode FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct OperatorCodeBuilder { + typedef OperatorCode Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_deprecated_builtin_code(int8_t deprecated_builtin_code) { @@ -10782,6 +11490,7 @@ struct OperatorT : public flatbuffers::NativeTable { struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef OperatorT NativeTableType; + typedef OperatorBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_OPCODE_INDEX = 4, VT_INPUTS = 6, @@ -11160,6 +11869,21 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const tflite::DynamicUpdateSliceOptions *builtin_options_as_DynamicUpdateSliceOptions() const { return builtin_options_type() == tflite::BuiltinOptions_DynamicUpdateSliceOptions ? static_cast(builtin_options()) : nullptr; } + const tflite::UnsortedSegmentProdOptions *builtin_options_as_UnsortedSegmentProdOptions() const { + return builtin_options_type() == tflite::BuiltinOptions_UnsortedSegmentProdOptions ? static_cast(builtin_options()) : nullptr; + } + const tflite::UnsortedSegmentMaxOptions *builtin_options_as_UnsortedSegmentMaxOptions() const { + return builtin_options_type() == tflite::BuiltinOptions_UnsortedSegmentMaxOptions ? static_cast(builtin_options()) : nullptr; + } + const tflite::UnsortedSegmentMinOptions *builtin_options_as_UnsortedSegmentMinOptions() const { + return builtin_options_type() == tflite::BuiltinOptions_UnsortedSegmentMinOptions ? static_cast(builtin_options()) : nullptr; + } + const tflite::UnsortedSegmentSumOptions *builtin_options_as_UnsortedSegmentSumOptions() const { + return builtin_options_type() == tflite::BuiltinOptions_UnsortedSegmentSumOptions ? static_cast(builtin_options()) : nullptr; + } + const tflite::ATan2Options *builtin_options_as_ATan2Options() const { + return builtin_options_type() == tflite::BuiltinOptions_ATan2Options ? static_cast(builtin_options()) : nullptr; + } const flatbuffers::Vector *custom_options() const { return GetPointer *>(VT_CUSTOM_OPTIONS); } @@ -11664,7 +12388,28 @@ template<> inline const tflite::DynamicUpdateSliceOptions *Operator::builtin_opt return builtin_options_as_DynamicUpdateSliceOptions(); } +template<> inline const tflite::UnsortedSegmentProdOptions *Operator::builtin_options_as() const { + return builtin_options_as_UnsortedSegmentProdOptions(); +} + +template<> inline const tflite::UnsortedSegmentMaxOptions *Operator::builtin_options_as() const { + return builtin_options_as_UnsortedSegmentMaxOptions(); +} + +template<> inline const tflite::UnsortedSegmentMinOptions *Operator::builtin_options_as() const { + return builtin_options_as_UnsortedSegmentMinOptions(); +} + +template<> inline const tflite::UnsortedSegmentSumOptions *Operator::builtin_options_as() const { + return builtin_options_as_UnsortedSegmentSumOptions(); +} + +template<> inline const tflite::ATan2Options *Operator::builtin_options_as() const { + return builtin_options_as_ATan2Options(); +} + struct OperatorBuilder { + typedef Operator Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_opcode_index(uint32_t opcode_index) { @@ -11774,6 +12519,7 @@ struct SubGraphT : public flatbuffers::NativeTable { struct SubGraph FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SubGraphT NativeTableType; + typedef SubGraphBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_TENSORS = 4, VT_INPUTS = 6, @@ -11818,6 +12564,7 @@ struct SubGraph FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SubGraphBuilder { + typedef SubGraph Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_tensors(flatbuffers::Offset>> tensors) { @@ -11895,6 +12642,7 @@ struct BufferT : public flatbuffers::NativeTable { struct Buffer FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef BufferT NativeTableType; + typedef BufferBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_DATA = 4 }; @@ -11913,6 +12661,7 @@ struct Buffer FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct BufferBuilder { + typedef Buffer Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_data(flatbuffers::Offset> data) { @@ -11961,6 +12710,7 @@ struct MetadataT : public flatbuffers::NativeTable { struct Metadata FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef MetadataT NativeTableType; + typedef MetadataBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_NAME = 4, VT_BUFFER = 6 @@ -11984,6 +12734,7 @@ struct Metadata FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct MetadataBuilder { + typedef Metadata Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_name(flatbuffers::Offset name) { @@ -12038,6 +12789,7 @@ struct TensorMapT : public flatbuffers::NativeTable { struct TensorMap FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef TensorMapT NativeTableType; + typedef TensorMapBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_NAME = 4, VT_TENSOR_INDEX = 6 @@ -12061,6 +12813,7 @@ struct TensorMap FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct TensorMapBuilder { + typedef TensorMap Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_name(flatbuffers::Offset name) { @@ -12117,6 +12870,7 @@ struct SignatureDefT : public flatbuffers::NativeTable { struct SignatureDef FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SignatureDefT NativeTableType; + typedef SignatureDefBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_INPUTS = 4, VT_OUTPUTS = 6, @@ -12154,6 +12908,7 @@ struct SignatureDef FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct SignatureDefBuilder { + typedef SignatureDef Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_inputs(flatbuffers::Offset>> inputs) { @@ -12230,6 +12985,7 @@ struct ModelT : public flatbuffers::NativeTable { struct Model FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef ModelT NativeTableType; + typedef ModelBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_VERSION = 4, VT_OPERATOR_CODES = 6, @@ -12294,6 +13050,7 @@ struct Model FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { }; struct ModelBuilder { + typedef Model Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_version(uint32_t version) { @@ -12386,15 +13143,15 @@ inline flatbuffers::Offset CreateModelDirect( flatbuffers::Offset CreateModel(flatbuffers::FlatBufferBuilder &_fbb, const ModelT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); inline CustomQuantizationT *CustomQuantization::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new CustomQuantizationT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new CustomQuantizationT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void CustomQuantization::UnPackTo(CustomQuantizationT *_o, const flatbuffers::resolver_function_t *_resolver) const { (void)_o; (void)_resolver; - { auto _e = custom(); if (_e) { _o->custom.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->custom.begin()); } } + { auto _e = custom(); if (_e) { _o->custom.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->custom[_i] = _e->Get(_i); } } } } inline flatbuffers::Offset CustomQuantization::Pack(flatbuffers::FlatBufferBuilder &_fbb, const CustomQuantizationT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -12413,9 +13170,9 @@ inline flatbuffers::Offset CreateCustomQuantization(flatbuff } inline QuantizationParametersT *QuantizationParameters::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new QuantizationParametersT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new QuantizationParametersT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void QuantizationParameters::UnPackTo(QuantizationParametersT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12457,9 +13214,9 @@ inline flatbuffers::Offset CreateQuantizationParameters( } inline Int32VectorT *Int32Vector::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new Int32VectorT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new Int32VectorT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Int32Vector::UnPackTo(Int32VectorT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12483,9 +13240,9 @@ inline flatbuffers::Offset CreateInt32Vector(flatbuffers::FlatBuffe } inline Uint16VectorT *Uint16Vector::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new Uint16VectorT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new Uint16VectorT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Uint16Vector::UnPackTo(Uint16VectorT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12510,15 +13267,15 @@ inline flatbuffers::Offset CreateUint16Vector(flatbuffers::FlatBuf } inline Uint8VectorT *Uint8Vector::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new Uint8VectorT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new Uint8VectorT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Uint8Vector::UnPackTo(Uint8VectorT *_o, const flatbuffers::resolver_function_t *_resolver) const { (void)_o; (void)_resolver; - { auto _e = values(); if (_e) { _o->values.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->values.begin()); } } + { auto _e = values(); if (_e) { _o->values.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->values[_i] = _e->Get(_i); } } } } inline flatbuffers::Offset Uint8Vector::Pack(flatbuffers::FlatBufferBuilder &_fbb, const Uint8VectorT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -12537,9 +13294,9 @@ inline flatbuffers::Offset CreateUint8Vector(flatbuffers::FlatBuffe } inline DimensionMetadataT *DimensionMetadata::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new DimensionMetadataT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new DimensionMetadataT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void DimensionMetadata::UnPackTo(DimensionMetadataT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12578,9 +13335,9 @@ inline flatbuffers::Offset CreateDimensionMetadata(flatbuffer } inline SparsityParametersT *SparsityParameters::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SparsityParametersT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SparsityParametersT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SparsityParameters::UnPackTo(SparsityParametersT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12610,9 +13367,9 @@ inline flatbuffers::Offset CreateSparsityParameters(flatbuff } inline TensorT *Tensor::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new TensorT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new TensorT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Tensor::UnPackTo(TensorT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12626,6 +13383,7 @@ inline void Tensor::UnPackTo(TensorT *_o, const flatbuffers::resolver_function_t { auto _e = is_variable(); _o->is_variable = _e; } { auto _e = sparsity(); if (_e) _o->sparsity = std::unique_ptr(_e->UnPack(_resolver)); } { auto _e = shape_signature(); if (_e) { _o->shape_signature.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->shape_signature[_i] = _e->Get(_i); } } } + { auto _e = has_rank(); _o->has_rank = _e; } } inline flatbuffers::Offset Tensor::Pack(flatbuffers::FlatBufferBuilder &_fbb, const TensorT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -12644,6 +13402,7 @@ inline flatbuffers::Offset CreateTensor(flatbuffers::FlatBufferBuilder & auto _is_variable = _o->is_variable; auto _sparsity = _o->sparsity ? CreateSparsityParameters(_fbb, _o->sparsity.get(), _rehasher) : 0; auto _shape_signature = _o->shape_signature.size() ? _fbb.CreateVector(_o->shape_signature) : 0; + auto _has_rank = _o->has_rank; return tflite::CreateTensor( _fbb, _shape, @@ -12653,13 +13412,14 @@ inline flatbuffers::Offset CreateTensor(flatbuffers::FlatBufferBuilder & _quantization, _is_variable, _sparsity, - _shape_signature); + _shape_signature, + _has_rank); } inline Conv2DOptionsT *Conv2DOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new Conv2DOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new Conv2DOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Conv2DOptions::UnPackTo(Conv2DOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12698,9 +13458,9 @@ inline flatbuffers::Offset CreateConv2DOptions(flatbuffers::FlatB } inline Conv3DOptionsT *Conv3DOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new Conv3DOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new Conv3DOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Conv3DOptions::UnPackTo(Conv3DOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12745,9 +13505,9 @@ inline flatbuffers::Offset CreateConv3DOptions(flatbuffers::FlatB } inline Pool2DOptionsT *Pool2DOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new Pool2DOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new Pool2DOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Pool2DOptions::UnPackTo(Pool2DOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12786,9 +13546,9 @@ inline flatbuffers::Offset CreatePool2DOptions(flatbuffers::FlatB } inline DepthwiseConv2DOptionsT *DepthwiseConv2DOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new DepthwiseConv2DOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new DepthwiseConv2DOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void DepthwiseConv2DOptions::UnPackTo(DepthwiseConv2DOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12830,9 +13590,9 @@ inline flatbuffers::Offset CreateDepthwiseConv2DOptions( } inline ConcatEmbeddingsOptionsT *ConcatEmbeddingsOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ConcatEmbeddingsOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ConcatEmbeddingsOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ConcatEmbeddingsOptions::UnPackTo(ConcatEmbeddingsOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12862,9 +13622,9 @@ inline flatbuffers::Offset CreateConcatEmbeddingsOption } inline LSHProjectionOptionsT *LSHProjectionOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LSHProjectionOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LSHProjectionOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LSHProjectionOptions::UnPackTo(LSHProjectionOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12888,9 +13648,9 @@ inline flatbuffers::Offset CreateLSHProjectionOptions(flat } inline SVDFOptionsT *SVDFOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SVDFOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SVDFOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SVDFOptions::UnPackTo(SVDFOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12920,9 +13680,9 @@ inline flatbuffers::Offset CreateSVDFOptions(flatbuffers::FlatBuffe } inline RNNOptionsT *RNNOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new RNNOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new RNNOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void RNNOptions::UnPackTo(RNNOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12949,9 +13709,9 @@ inline flatbuffers::Offset CreateRNNOptions(flatbuffers::FlatBufferB } inline SequenceRNNOptionsT *SequenceRNNOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SequenceRNNOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SequenceRNNOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SequenceRNNOptions::UnPackTo(SequenceRNNOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -12981,9 +13741,9 @@ inline flatbuffers::Offset CreateSequenceRNNOptions(flatbuff } inline BidirectionalSequenceRNNOptionsT *BidirectionalSequenceRNNOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new BidirectionalSequenceRNNOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new BidirectionalSequenceRNNOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void BidirectionalSequenceRNNOptions::UnPackTo(BidirectionalSequenceRNNOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13016,9 +13776,9 @@ inline flatbuffers::Offset CreateBidirectionalS } inline FullyConnectedOptionsT *FullyConnectedOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new FullyConnectedOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new FullyConnectedOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void FullyConnectedOptions::UnPackTo(FullyConnectedOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13051,9 +13811,9 @@ inline flatbuffers::Offset CreateFullyConnectedOptions(fl } inline SoftmaxOptionsT *SoftmaxOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SoftmaxOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SoftmaxOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SoftmaxOptions::UnPackTo(SoftmaxOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13077,9 +13837,9 @@ inline flatbuffers::Offset CreateSoftmaxOptions(flatbuffers::Fla } inline ConcatenationOptionsT *ConcatenationOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ConcatenationOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ConcatenationOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ConcatenationOptions::UnPackTo(ConcatenationOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13106,9 +13866,9 @@ inline flatbuffers::Offset CreateConcatenationOptions(flat } inline AddOptionsT *AddOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new AddOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new AddOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void AddOptions::UnPackTo(AddOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13135,9 +13895,9 @@ inline flatbuffers::Offset CreateAddOptions(flatbuffers::FlatBufferB } inline MulOptionsT *MulOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new MulOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new MulOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void MulOptions::UnPackTo(MulOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13161,9 +13921,9 @@ inline flatbuffers::Offset CreateMulOptions(flatbuffers::FlatBufferB } inline L2NormOptionsT *L2NormOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new L2NormOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new L2NormOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void L2NormOptions::UnPackTo(L2NormOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13187,9 +13947,9 @@ inline flatbuffers::Offset CreateL2NormOptions(flatbuffers::FlatB } inline LocalResponseNormalizationOptionsT *LocalResponseNormalizationOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LocalResponseNormalizationOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LocalResponseNormalizationOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LocalResponseNormalizationOptions::UnPackTo(LocalResponseNormalizationOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13222,9 +13982,9 @@ inline flatbuffers::Offset CreateLocalRespons } inline LSTMOptionsT *LSTMOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LSTMOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LSTMOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LSTMOptions::UnPackTo(LSTMOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13260,9 +14020,9 @@ inline flatbuffers::Offset CreateLSTMOptions(flatbuffers::FlatBuffe } inline UnidirectionalSequenceLSTMOptionsT *UnidirectionalSequenceLSTMOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new UnidirectionalSequenceLSTMOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new UnidirectionalSequenceLSTMOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void UnidirectionalSequenceLSTMOptions::UnPackTo(UnidirectionalSequenceLSTMOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13298,9 +14058,9 @@ inline flatbuffers::Offset CreateUnidirection } inline BidirectionalSequenceLSTMOptionsT *BidirectionalSequenceLSTMOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new BidirectionalSequenceLSTMOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new BidirectionalSequenceLSTMOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void BidirectionalSequenceLSTMOptions::UnPackTo(BidirectionalSequenceLSTMOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13339,9 +14099,9 @@ inline flatbuffers::Offset CreateBidirectional } inline ResizeBilinearOptionsT *ResizeBilinearOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ResizeBilinearOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ResizeBilinearOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ResizeBilinearOptions::UnPackTo(ResizeBilinearOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13368,9 +14128,9 @@ inline flatbuffers::Offset CreateResizeBilinearOptions(fl } inline ResizeNearestNeighborOptionsT *ResizeNearestNeighborOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ResizeNearestNeighborOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ResizeNearestNeighborOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ResizeNearestNeighborOptions::UnPackTo(ResizeNearestNeighborOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13397,9 +14157,9 @@ inline flatbuffers::Offset CreateResizeNearestNeig } inline CallOptionsT *CallOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new CallOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new CallOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void CallOptions::UnPackTo(CallOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13423,9 +14183,9 @@ inline flatbuffers::Offset CreateCallOptions(flatbuffers::FlatBuffe } inline PadOptionsT *PadOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new PadOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new PadOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void PadOptions::UnPackTo(PadOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13446,9 +14206,9 @@ inline flatbuffers::Offset CreatePadOptions(flatbuffers::FlatBufferB } inline PadV2OptionsT *PadV2Options::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new PadV2OptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new PadV2OptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void PadV2Options::UnPackTo(PadV2OptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13469,9 +14229,9 @@ inline flatbuffers::Offset CreatePadV2Options(flatbuffers::FlatBuf } inline ReshapeOptionsT *ReshapeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ReshapeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ReshapeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ReshapeOptions::UnPackTo(ReshapeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13495,9 +14255,9 @@ inline flatbuffers::Offset CreateReshapeOptions(flatbuffers::Fla } inline SpaceToBatchNDOptionsT *SpaceToBatchNDOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SpaceToBatchNDOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SpaceToBatchNDOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SpaceToBatchNDOptions::UnPackTo(SpaceToBatchNDOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13518,9 +14278,9 @@ inline flatbuffers::Offset CreateSpaceToBatchNDOptions(fl } inline BatchToSpaceNDOptionsT *BatchToSpaceNDOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new BatchToSpaceNDOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new BatchToSpaceNDOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void BatchToSpaceNDOptions::UnPackTo(BatchToSpaceNDOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13541,9 +14301,9 @@ inline flatbuffers::Offset CreateBatchToSpaceNDOptions(fl } inline SkipGramOptionsT *SkipGramOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SkipGramOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SkipGramOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SkipGramOptions::UnPackTo(SkipGramOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13573,9 +14333,9 @@ inline flatbuffers::Offset CreateSkipGramOptions(flatbuffers::F } inline SpaceToDepthOptionsT *SpaceToDepthOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SpaceToDepthOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SpaceToDepthOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SpaceToDepthOptions::UnPackTo(SpaceToDepthOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13599,9 +14359,9 @@ inline flatbuffers::Offset CreateSpaceToDepthOptions(flatbu } inline DepthToSpaceOptionsT *DepthToSpaceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new DepthToSpaceOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new DepthToSpaceOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void DepthToSpaceOptions::UnPackTo(DepthToSpaceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13625,9 +14385,9 @@ inline flatbuffers::Offset CreateDepthToSpaceOptions(flatbu } inline SubOptionsT *SubOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SubOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SubOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SubOptions::UnPackTo(SubOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13654,9 +14414,9 @@ inline flatbuffers::Offset CreateSubOptions(flatbuffers::FlatBufferB } inline DivOptionsT *DivOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new DivOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new DivOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void DivOptions::UnPackTo(DivOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13680,9 +14440,9 @@ inline flatbuffers::Offset CreateDivOptions(flatbuffers::FlatBufferB } inline TopKV2OptionsT *TopKV2Options::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new TopKV2OptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new TopKV2OptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void TopKV2Options::UnPackTo(TopKV2OptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13703,9 +14463,9 @@ inline flatbuffers::Offset CreateTopKV2Options(flatbuffers::FlatB } inline EmbeddingLookupSparseOptionsT *EmbeddingLookupSparseOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new EmbeddingLookupSparseOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new EmbeddingLookupSparseOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void EmbeddingLookupSparseOptions::UnPackTo(EmbeddingLookupSparseOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13729,9 +14489,9 @@ inline flatbuffers::Offset CreateEmbeddingLookupSp } inline GatherOptionsT *GatherOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new GatherOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new GatherOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void GatherOptions::UnPackTo(GatherOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13758,9 +14518,9 @@ inline flatbuffers::Offset CreateGatherOptions(flatbuffers::FlatB } inline TransposeOptionsT *TransposeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new TransposeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new TransposeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void TransposeOptions::UnPackTo(TransposeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13781,9 +14541,9 @@ inline flatbuffers::Offset CreateTransposeOptions(flatbuffers: } inline ExpOptionsT *ExpOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ExpOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ExpOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ExpOptions::UnPackTo(ExpOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13804,9 +14564,9 @@ inline flatbuffers::Offset CreateExpOptions(flatbuffers::FlatBufferB } inline CosOptionsT *CosOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new CosOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new CosOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void CosOptions::UnPackTo(CosOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13827,9 +14587,9 @@ inline flatbuffers::Offset CreateCosOptions(flatbuffers::FlatBufferB } inline ReducerOptionsT *ReducerOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ReducerOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ReducerOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ReducerOptions::UnPackTo(ReducerOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13853,9 +14613,9 @@ inline flatbuffers::Offset CreateReducerOptions(flatbuffers::Fla } inline SqueezeOptionsT *SqueezeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SqueezeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SqueezeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SqueezeOptions::UnPackTo(SqueezeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13879,9 +14639,9 @@ inline flatbuffers::Offset CreateSqueezeOptions(flatbuffers::Fla } inline SplitOptionsT *SplitOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SplitOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SplitOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SplitOptions::UnPackTo(SplitOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13905,9 +14665,9 @@ inline flatbuffers::Offset CreateSplitOptions(flatbuffers::FlatBuf } inline SplitVOptionsT *SplitVOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SplitVOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SplitVOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SplitVOptions::UnPackTo(SplitVOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13931,9 +14691,9 @@ inline flatbuffers::Offset CreateSplitVOptions(flatbuffers::FlatB } inline StridedSliceOptionsT *StridedSliceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new StridedSliceOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new StridedSliceOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void StridedSliceOptions::UnPackTo(StridedSliceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13969,9 +14729,9 @@ inline flatbuffers::Offset CreateStridedSliceOptions(flatbu } inline LogSoftmaxOptionsT *LogSoftmaxOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LogSoftmaxOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LogSoftmaxOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LogSoftmaxOptions::UnPackTo(LogSoftmaxOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -13992,9 +14752,9 @@ inline flatbuffers::Offset CreateLogSoftmaxOptions(flatbuffer } inline CastOptionsT *CastOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new CastOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new CastOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void CastOptions::UnPackTo(CastOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14021,9 +14781,9 @@ inline flatbuffers::Offset CreateCastOptions(flatbuffers::FlatBuffe } inline DequantizeOptionsT *DequantizeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new DequantizeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new DequantizeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void DequantizeOptions::UnPackTo(DequantizeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14044,9 +14804,9 @@ inline flatbuffers::Offset CreateDequantizeOptions(flatbuffer } inline MaximumMinimumOptionsT *MaximumMinimumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new MaximumMinimumOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new MaximumMinimumOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void MaximumMinimumOptions::UnPackTo(MaximumMinimumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14067,9 +14827,9 @@ inline flatbuffers::Offset CreateMaximumMinimumOptions(fl } inline TileOptionsT *TileOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new TileOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new TileOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void TileOptions::UnPackTo(TileOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14090,9 +14850,9 @@ inline flatbuffers::Offset CreateTileOptions(flatbuffers::FlatBuffe } inline ArgMaxOptionsT *ArgMaxOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ArgMaxOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ArgMaxOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ArgMaxOptions::UnPackTo(ArgMaxOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14116,9 +14876,9 @@ inline flatbuffers::Offset CreateArgMaxOptions(flatbuffers::FlatB } inline ArgMinOptionsT *ArgMinOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ArgMinOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ArgMinOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ArgMinOptions::UnPackTo(ArgMinOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14142,9 +14902,9 @@ inline flatbuffers::Offset CreateArgMinOptions(flatbuffers::FlatB } inline GreaterOptionsT *GreaterOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new GreaterOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new GreaterOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void GreaterOptions::UnPackTo(GreaterOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14165,9 +14925,9 @@ inline flatbuffers::Offset CreateGreaterOptions(flatbuffers::Fla } inline GreaterEqualOptionsT *GreaterEqualOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new GreaterEqualOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new GreaterEqualOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void GreaterEqualOptions::UnPackTo(GreaterEqualOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14188,9 +14948,9 @@ inline flatbuffers::Offset CreateGreaterEqualOptions(flatbu } inline LessOptionsT *LessOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LessOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LessOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LessOptions::UnPackTo(LessOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14211,9 +14971,9 @@ inline flatbuffers::Offset CreateLessOptions(flatbuffers::FlatBuffe } inline LessEqualOptionsT *LessEqualOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LessEqualOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LessEqualOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LessEqualOptions::UnPackTo(LessEqualOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14234,9 +14994,9 @@ inline flatbuffers::Offset CreateLessEqualOptions(flatbuffers: } inline NegOptionsT *NegOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new NegOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new NegOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void NegOptions::UnPackTo(NegOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14257,9 +15017,9 @@ inline flatbuffers::Offset CreateNegOptions(flatbuffers::FlatBufferB } inline SelectOptionsT *SelectOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SelectOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SelectOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SelectOptions::UnPackTo(SelectOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14280,9 +15040,9 @@ inline flatbuffers::Offset CreateSelectOptions(flatbuffers::FlatB } inline SliceOptionsT *SliceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SliceOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SliceOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SliceOptions::UnPackTo(SliceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14303,9 +15063,9 @@ inline flatbuffers::Offset CreateSliceOptions(flatbuffers::FlatBuf } inline TransposeConvOptionsT *TransposeConvOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new TransposeConvOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new TransposeConvOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void TransposeConvOptions::UnPackTo(TransposeConvOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14335,9 +15095,9 @@ inline flatbuffers::Offset CreateTransposeConvOptions(flat } inline ExpandDimsOptionsT *ExpandDimsOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ExpandDimsOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ExpandDimsOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ExpandDimsOptions::UnPackTo(ExpandDimsOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14358,9 +15118,9 @@ inline flatbuffers::Offset CreateExpandDimsOptions(flatbuffer } inline SparseToDenseOptionsT *SparseToDenseOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SparseToDenseOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SparseToDenseOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SparseToDenseOptions::UnPackTo(SparseToDenseOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14384,9 +15144,9 @@ inline flatbuffers::Offset CreateSparseToDenseOptions(flat } inline EqualOptionsT *EqualOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new EqualOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new EqualOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void EqualOptions::UnPackTo(EqualOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14407,9 +15167,9 @@ inline flatbuffers::Offset CreateEqualOptions(flatbuffers::FlatBuf } inline NotEqualOptionsT *NotEqualOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new NotEqualOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new NotEqualOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void NotEqualOptions::UnPackTo(NotEqualOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14430,9 +15190,9 @@ inline flatbuffers::Offset CreateNotEqualOptions(flatbuffers::F } inline ShapeOptionsT *ShapeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ShapeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ShapeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ShapeOptions::UnPackTo(ShapeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14456,9 +15216,9 @@ inline flatbuffers::Offset CreateShapeOptions(flatbuffers::FlatBuf } inline RankOptionsT *RankOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new RankOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new RankOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void RankOptions::UnPackTo(RankOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14479,9 +15239,9 @@ inline flatbuffers::Offset CreateRankOptions(flatbuffers::FlatBuffe } inline PowOptionsT *PowOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new PowOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new PowOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void PowOptions::UnPackTo(PowOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14502,9 +15262,9 @@ inline flatbuffers::Offset CreatePowOptions(flatbuffers::FlatBufferB } inline FakeQuantOptionsT *FakeQuantOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new FakeQuantOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new FakeQuantOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void FakeQuantOptions::UnPackTo(FakeQuantOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14537,9 +15297,9 @@ inline flatbuffers::Offset CreateFakeQuantOptions(flatbuffers: } inline PackOptionsT *PackOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new PackOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new PackOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void PackOptions::UnPackTo(PackOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14566,9 +15326,9 @@ inline flatbuffers::Offset CreatePackOptions(flatbuffers::FlatBuffe } inline LogicalOrOptionsT *LogicalOrOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LogicalOrOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LogicalOrOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LogicalOrOptions::UnPackTo(LogicalOrOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14589,9 +15349,9 @@ inline flatbuffers::Offset CreateLogicalOrOptions(flatbuffers: } inline OneHotOptionsT *OneHotOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new OneHotOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new OneHotOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void OneHotOptions::UnPackTo(OneHotOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14615,9 +15375,9 @@ inline flatbuffers::Offset CreateOneHotOptions(flatbuffers::FlatB } inline AbsOptionsT *AbsOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new AbsOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new AbsOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void AbsOptions::UnPackTo(AbsOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14638,9 +15398,9 @@ inline flatbuffers::Offset CreateAbsOptions(flatbuffers::FlatBufferB } inline HardSwishOptionsT *HardSwishOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new HardSwishOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new HardSwishOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void HardSwishOptions::UnPackTo(HardSwishOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14661,9 +15421,9 @@ inline flatbuffers::Offset CreateHardSwishOptions(flatbuffers: } inline LogicalAndOptionsT *LogicalAndOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LogicalAndOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LogicalAndOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LogicalAndOptions::UnPackTo(LogicalAndOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14684,9 +15444,9 @@ inline flatbuffers::Offset CreateLogicalAndOptions(flatbuffer } inline LogicalNotOptionsT *LogicalNotOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LogicalNotOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LogicalNotOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LogicalNotOptions::UnPackTo(LogicalNotOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14707,9 +15467,9 @@ inline flatbuffers::Offset CreateLogicalNotOptions(flatbuffer } inline UnpackOptionsT *UnpackOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new UnpackOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new UnpackOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void UnpackOptions::UnPackTo(UnpackOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14736,9 +15496,9 @@ inline flatbuffers::Offset CreateUnpackOptions(flatbuffers::FlatB } inline FloorDivOptionsT *FloorDivOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new FloorDivOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new FloorDivOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void FloorDivOptions::UnPackTo(FloorDivOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14759,9 +15519,9 @@ inline flatbuffers::Offset CreateFloorDivOptions(flatbuffers::F } inline SquareOptionsT *SquareOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SquareOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SquareOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SquareOptions::UnPackTo(SquareOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14782,9 +15542,9 @@ inline flatbuffers::Offset CreateSquareOptions(flatbuffers::FlatB } inline ZerosLikeOptionsT *ZerosLikeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ZerosLikeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ZerosLikeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ZerosLikeOptions::UnPackTo(ZerosLikeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14805,9 +15565,9 @@ inline flatbuffers::Offset CreateZerosLikeOptions(flatbuffers: } inline FillOptionsT *FillOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new FillOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new FillOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void FillOptions::UnPackTo(FillOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14828,9 +15588,9 @@ inline flatbuffers::Offset CreateFillOptions(flatbuffers::FlatBuffe } inline FloorModOptionsT *FloorModOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new FloorModOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new FloorModOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void FloorModOptions::UnPackTo(FloorModOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14851,9 +15611,9 @@ inline flatbuffers::Offset CreateFloorModOptions(flatbuffers::F } inline RangeOptionsT *RangeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new RangeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new RangeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void RangeOptions::UnPackTo(RangeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14874,9 +15634,9 @@ inline flatbuffers::Offset CreateRangeOptions(flatbuffers::FlatBuf } inline LeakyReluOptionsT *LeakyReluOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new LeakyReluOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new LeakyReluOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void LeakyReluOptions::UnPackTo(LeakyReluOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14900,9 +15660,9 @@ inline flatbuffers::Offset CreateLeakyReluOptions(flatbuffers: } inline SquaredDifferenceOptionsT *SquaredDifferenceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SquaredDifferenceOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SquaredDifferenceOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SquaredDifferenceOptions::UnPackTo(SquaredDifferenceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14923,9 +15683,9 @@ inline flatbuffers::Offset CreateSquaredDifferenceOpti } inline MirrorPadOptionsT *MirrorPadOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new MirrorPadOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new MirrorPadOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void MirrorPadOptions::UnPackTo(MirrorPadOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14949,9 +15709,9 @@ inline flatbuffers::Offset CreateMirrorPadOptions(flatbuffers: } inline UniqueOptionsT *UniqueOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new UniqueOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new UniqueOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void UniqueOptions::UnPackTo(UniqueOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14975,9 +15735,9 @@ inline flatbuffers::Offset CreateUniqueOptions(flatbuffers::FlatB } inline ReverseV2OptionsT *ReverseV2Options::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ReverseV2OptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ReverseV2OptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ReverseV2Options::UnPackTo(ReverseV2OptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -14998,9 +15758,9 @@ inline flatbuffers::Offset CreateReverseV2Options(flatbuffers: } inline AddNOptionsT *AddNOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new AddNOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new AddNOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void AddNOptions::UnPackTo(AddNOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15021,9 +15781,9 @@ inline flatbuffers::Offset CreateAddNOptions(flatbuffers::FlatBuffe } inline GatherNdOptionsT *GatherNdOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new GatherNdOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new GatherNdOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void GatherNdOptions::UnPackTo(GatherNdOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15044,9 +15804,9 @@ inline flatbuffers::Offset CreateGatherNdOptions(flatbuffers::F } inline WhereOptionsT *WhereOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new WhereOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new WhereOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void WhereOptions::UnPackTo(WhereOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15067,9 +15827,9 @@ inline flatbuffers::Offset CreateWhereOptions(flatbuffers::FlatBuf } inline ReverseSequenceOptionsT *ReverseSequenceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ReverseSequenceOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ReverseSequenceOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ReverseSequenceOptions::UnPackTo(ReverseSequenceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15096,9 +15856,9 @@ inline flatbuffers::Offset CreateReverseSequenceOptions( } inline MatrixDiagOptionsT *MatrixDiagOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new MatrixDiagOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new MatrixDiagOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void MatrixDiagOptions::UnPackTo(MatrixDiagOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15119,9 +15879,9 @@ inline flatbuffers::Offset CreateMatrixDiagOptions(flatbuffer } inline QuantizeOptionsT *QuantizeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new QuantizeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new QuantizeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void QuantizeOptions::UnPackTo(QuantizeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15142,9 +15902,9 @@ inline flatbuffers::Offset CreateQuantizeOptions(flatbuffers::F } inline MatrixSetDiagOptionsT *MatrixSetDiagOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new MatrixSetDiagOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new MatrixSetDiagOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void MatrixSetDiagOptions::UnPackTo(MatrixSetDiagOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15165,9 +15925,9 @@ inline flatbuffers::Offset CreateMatrixSetDiagOptions(flat } inline IfOptionsT *IfOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new IfOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new IfOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void IfOptions::UnPackTo(IfOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15194,9 +15954,9 @@ inline flatbuffers::Offset CreateIfOptions(flatbuffers::FlatBufferBui } inline CallOnceOptionsT *CallOnceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new CallOnceOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new CallOnceOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void CallOnceOptions::UnPackTo(CallOnceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15220,9 +15980,9 @@ inline flatbuffers::Offset CreateCallOnceOptions(flatbuffers::F } inline WhileOptionsT *WhileOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new WhileOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new WhileOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void WhileOptions::UnPackTo(WhileOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15249,9 +16009,9 @@ inline flatbuffers::Offset CreateWhileOptions(flatbuffers::FlatBuf } inline NonMaxSuppressionV4OptionsT *NonMaxSuppressionV4Options::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new NonMaxSuppressionV4OptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new NonMaxSuppressionV4OptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void NonMaxSuppressionV4Options::UnPackTo(NonMaxSuppressionV4OptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15272,9 +16032,9 @@ inline flatbuffers::Offset CreateNonMaxSuppressionV4 } inline NonMaxSuppressionV5OptionsT *NonMaxSuppressionV5Options::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new NonMaxSuppressionV5OptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new NonMaxSuppressionV5OptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void NonMaxSuppressionV5Options::UnPackTo(NonMaxSuppressionV5OptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15295,9 +16055,9 @@ inline flatbuffers::Offset CreateNonMaxSuppressionV5 } inline ScatterNdOptionsT *ScatterNdOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ScatterNdOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ScatterNdOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ScatterNdOptions::UnPackTo(ScatterNdOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15318,9 +16078,9 @@ inline flatbuffers::Offset CreateScatterNdOptions(flatbuffers: } inline SelectV2OptionsT *SelectV2Options::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SelectV2OptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SelectV2OptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SelectV2Options::UnPackTo(SelectV2OptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15341,9 +16101,9 @@ inline flatbuffers::Offset CreateSelectV2Options(flatbuffers::F } inline DensifyOptionsT *DensifyOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new DensifyOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new DensifyOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void DensifyOptions::UnPackTo(DensifyOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15364,9 +16124,9 @@ inline flatbuffers::Offset CreateDensifyOptions(flatbuffers::Fla } inline SegmentSumOptionsT *SegmentSumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SegmentSumOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SegmentSumOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SegmentSumOptions::UnPackTo(SegmentSumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15387,9 +16147,9 @@ inline flatbuffers::Offset CreateSegmentSumOptions(flatbuffer } inline BatchMatMulOptionsT *BatchMatMulOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new BatchMatMulOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new BatchMatMulOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void BatchMatMulOptions::UnPackTo(BatchMatMulOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15419,9 +16179,9 @@ inline flatbuffers::Offset CreateBatchMatMulOptions(flatbuff } inline CumsumOptionsT *CumsumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new CumsumOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new CumsumOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void CumsumOptions::UnPackTo(CumsumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15448,9 +16208,9 @@ inline flatbuffers::Offset CreateCumsumOptions(flatbuffers::FlatB } inline BroadcastToOptionsT *BroadcastToOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new BroadcastToOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new BroadcastToOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void BroadcastToOptions::UnPackTo(BroadcastToOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15471,9 +16231,9 @@ inline flatbuffers::Offset CreateBroadcastToOptions(flatbuff } inline Rfft2dOptionsT *Rfft2dOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new Rfft2dOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new Rfft2dOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Rfft2dOptions::UnPackTo(Rfft2dOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15494,9 +16254,9 @@ inline flatbuffers::Offset CreateRfft2dOptions(flatbuffers::FlatB } inline HashtableOptionsT *HashtableOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new HashtableOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new HashtableOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void HashtableOptions::UnPackTo(HashtableOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15526,9 +16286,9 @@ inline flatbuffers::Offset CreateHashtableOptions(flatbuffers: } inline HashtableFindOptionsT *HashtableFindOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new HashtableFindOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new HashtableFindOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void HashtableFindOptions::UnPackTo(HashtableFindOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15549,9 +16309,9 @@ inline flatbuffers::Offset CreateHashtableFindOptions(flat } inline HashtableImportOptionsT *HashtableImportOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new HashtableImportOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new HashtableImportOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void HashtableImportOptions::UnPackTo(HashtableImportOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15572,9 +16332,9 @@ inline flatbuffers::Offset CreateHashtableImportOptions( } inline HashtableSizeOptionsT *HashtableSizeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new HashtableSizeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new HashtableSizeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void HashtableSizeOptions::UnPackTo(HashtableSizeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15595,9 +16355,9 @@ inline flatbuffers::Offset CreateHashtableSizeOptions(flat } inline VarHandleOptionsT *VarHandleOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new VarHandleOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new VarHandleOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void VarHandleOptions::UnPackTo(VarHandleOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15624,9 +16384,9 @@ inline flatbuffers::Offset CreateVarHandleOptions(flatbuffers: } inline ReadVariableOptionsT *ReadVariableOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ReadVariableOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ReadVariableOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void ReadVariableOptions::UnPackTo(ReadVariableOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15647,9 +16407,9 @@ inline flatbuffers::Offset CreateReadVariableOptions(flatbu } inline AssignVariableOptionsT *AssignVariableOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new AssignVariableOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new AssignVariableOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void AssignVariableOptions::UnPackTo(AssignVariableOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15670,9 +16430,9 @@ inline flatbuffers::Offset CreateAssignVariableOptions(fl } inline RandomOptionsT *RandomOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new RandomOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new RandomOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void RandomOptions::UnPackTo(RandomOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15699,9 +16459,9 @@ inline flatbuffers::Offset CreateRandomOptions(flatbuffers::FlatB } inline BucketizeOptionsT *BucketizeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new BucketizeOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new BucketizeOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void BucketizeOptions::UnPackTo(BucketizeOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15725,9 +16485,9 @@ inline flatbuffers::Offset CreateBucketizeOptions(flatbuffers: } inline GeluOptionsT *GeluOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new GeluOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new GeluOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void GeluOptions::UnPackTo(GeluOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15751,9 +16511,9 @@ inline flatbuffers::Offset CreateGeluOptions(flatbuffers::FlatBuffe } inline DynamicUpdateSliceOptionsT *DynamicUpdateSliceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new DynamicUpdateSliceOptionsT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new DynamicUpdateSliceOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void DynamicUpdateSliceOptions::UnPackTo(DynamicUpdateSliceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15773,10 +16533,125 @@ inline flatbuffers::Offset CreateDynamicUpdateSliceOp _fbb); } +inline UnsortedSegmentProdOptionsT *UnsortedSegmentProdOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + std::unique_ptr _o = std::unique_ptr(new UnsortedSegmentProdOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void UnsortedSegmentProdOptions::UnPackTo(UnsortedSegmentProdOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset UnsortedSegmentProdOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentProdOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateUnsortedSegmentProdOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateUnsortedSegmentProdOptions(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentProdOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const UnsortedSegmentProdOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateUnsortedSegmentProdOptions( + _fbb); +} + +inline UnsortedSegmentMaxOptionsT *UnsortedSegmentMaxOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + std::unique_ptr _o = std::unique_ptr(new UnsortedSegmentMaxOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void UnsortedSegmentMaxOptions::UnPackTo(UnsortedSegmentMaxOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset UnsortedSegmentMaxOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentMaxOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateUnsortedSegmentMaxOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateUnsortedSegmentMaxOptions(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentMaxOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const UnsortedSegmentMaxOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateUnsortedSegmentMaxOptions( + _fbb); +} + +inline UnsortedSegmentSumOptionsT *UnsortedSegmentSumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + std::unique_ptr _o = std::unique_ptr(new UnsortedSegmentSumOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void UnsortedSegmentSumOptions::UnPackTo(UnsortedSegmentSumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset UnsortedSegmentSumOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentSumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateUnsortedSegmentSumOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateUnsortedSegmentSumOptions(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentSumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const UnsortedSegmentSumOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateUnsortedSegmentSumOptions( + _fbb); +} + +inline ATan2OptionsT *ATan2Options::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + std::unique_ptr _o = std::unique_ptr(new ATan2OptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void ATan2Options::UnPackTo(ATan2OptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset ATan2Options::Pack(flatbuffers::FlatBufferBuilder &_fbb, const ATan2OptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateATan2Options(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateATan2Options(flatbuffers::FlatBufferBuilder &_fbb, const ATan2OptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const ATan2OptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateATan2Options( + _fbb); +} + +inline UnsortedSegmentMinOptionsT *UnsortedSegmentMinOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + std::unique_ptr _o = std::unique_ptr(new UnsortedSegmentMinOptionsT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void UnsortedSegmentMinOptions::UnPackTo(UnsortedSegmentMinOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset UnsortedSegmentMinOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentMinOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateUnsortedSegmentMinOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateUnsortedSegmentMinOptions(flatbuffers::FlatBufferBuilder &_fbb, const UnsortedSegmentMinOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const UnsortedSegmentMinOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateUnsortedSegmentMinOptions( + _fbb); +} + inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new OperatorCodeT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new OperatorCodeT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void OperatorCode::UnPackTo(OperatorCodeT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15809,9 +16684,9 @@ inline flatbuffers::Offset CreateOperatorCode(flatbuffers::FlatBuf } inline OperatorT *Operator::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new OperatorT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new OperatorT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Operator::UnPackTo(OperatorT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15822,7 +16697,7 @@ inline void Operator::UnPackTo(OperatorT *_o, const flatbuffers::resolver_functi { auto _e = outputs(); if (_e) { _o->outputs.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->outputs[_i] = _e->Get(_i); } } } { auto _e = builtin_options_type(); _o->builtin_options.type = _e; } { auto _e = builtin_options(); if (_e) _o->builtin_options.value = tflite::BuiltinOptionsUnion::UnPack(_e, builtin_options_type(), _resolver); } - { auto _e = custom_options(); if (_e) { _o->custom_options.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->custom_options.begin()); } } + { auto _e = custom_options(); if (_e) { _o->custom_options.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->custom_options[_i] = _e->Get(_i); } } } { auto _e = custom_options_format(); _o->custom_options_format = _e; } { auto _e = mutating_variable_inputs(); if (_e) { _o->mutating_variable_inputs.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->mutating_variable_inputs[_i] = _e->Get(_i) != 0; } } } { auto _e = intermediates(); if (_e) { _o->intermediates.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->intermediates[_i] = _e->Get(_i); } } } @@ -15859,9 +16734,9 @@ inline flatbuffers::Offset CreateOperator(flatbuffers::FlatBufferBuild } inline SubGraphT *SubGraph::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SubGraphT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SubGraphT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SubGraph::UnPackTo(SubGraphT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15897,15 +16772,15 @@ inline flatbuffers::Offset CreateSubGraph(flatbuffers::FlatBufferBuild } inline BufferT *Buffer::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new BufferT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new BufferT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Buffer::UnPackTo(BufferT *_o, const flatbuffers::resolver_function_t *_resolver) const { (void)_o; (void)_resolver; - { auto _e = data(); if (_e) { _o->data.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->data.begin()); } } + { auto _e = data(); if (_e) { _o->data.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->data[_i] = _e->Get(_i); } } } } inline flatbuffers::Offset Buffer::Pack(flatbuffers::FlatBufferBuilder &_fbb, const BufferT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -15924,9 +16799,9 @@ inline flatbuffers::Offset CreateBuffer(flatbuffers::FlatBufferBuilder & } inline MetadataT *Metadata::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new MetadataT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new MetadataT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Metadata::UnPackTo(MetadataT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15953,9 +16828,9 @@ inline flatbuffers::Offset CreateMetadata(flatbuffers::FlatBufferBuild } inline TensorMapT *TensorMap::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new TensorMapT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new TensorMapT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void TensorMap::UnPackTo(TensorMapT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -15982,9 +16857,9 @@ inline flatbuffers::Offset CreateTensorMap(flatbuffers::FlatBufferBui } inline SignatureDefT *SignatureDef::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new SignatureDefT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new SignatureDefT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void SignatureDef::UnPackTo(SignatureDefT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -16017,9 +16892,9 @@ inline flatbuffers::Offset CreateSignatureDef(flatbuffers::FlatBuf } inline ModelT *Model::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new ModelT(); - UnPackTo(_o, _resolver); - return _o; + std::unique_ptr _o = std::unique_ptr(new ModelT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); } inline void Model::UnPackTo(ModelT *_o, const flatbuffers::resolver_function_t *_resolver) const { @@ -16108,7 +16983,7 @@ inline flatbuffers::Offset QuantizationDetailsUnion::Pack(flatbuffers::Fla } } -inline QuantizationDetailsUnion::QuantizationDetailsUnion(const QuantizationDetailsUnion &u) FLATBUFFERS_NOEXCEPT : type(u.type), value(nullptr) { +inline QuantizationDetailsUnion::QuantizationDetailsUnion(const QuantizationDetailsUnion &u) : type(u.type), value(nullptr) { switch (type) { case QuantizationDetails_CustomQuantization: { value = new tflite::CustomQuantizationT(*reinterpret_cast(u.value)); @@ -16201,7 +17076,7 @@ inline flatbuffers::Offset SparseIndexVectorUnion::Pack(flatbuffers::FlatB } } -inline SparseIndexVectorUnion::SparseIndexVectorUnion(const SparseIndexVectorUnion &u) FLATBUFFERS_NOEXCEPT : type(u.type), value(nullptr) { +inline SparseIndexVectorUnion::SparseIndexVectorUnion(const SparseIndexVectorUnion &u) : type(u.type), value(nullptr) { switch (type) { case SparseIndexVector_Int32Vector: { value = new tflite::Int32VectorT(*reinterpret_cast(u.value)); @@ -16716,6 +17591,26 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } + case BuiltinOptions_UnsortedSegmentProdOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_UnsortedSegmentMaxOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_UnsortedSegmentMinOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_UnsortedSegmentSumOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_ATan2Options: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } default: return true; } } @@ -17202,6 +18097,26 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } + case BuiltinOptions_UnsortedSegmentProdOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_UnsortedSegmentMaxOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_UnsortedSegmentMinOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_UnsortedSegmentSumOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_ATan2Options: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } default: return nullptr; } } @@ -17676,11 +18591,31 @@ inline flatbuffers::Offset BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff auto ptr = reinterpret_cast(value); return CreateDynamicUpdateSliceOptions(_fbb, ptr, _rehasher).Union(); } + case BuiltinOptions_UnsortedSegmentProdOptions: { + auto ptr = reinterpret_cast(value); + return CreateUnsortedSegmentProdOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_UnsortedSegmentMaxOptions: { + auto ptr = reinterpret_cast(value); + return CreateUnsortedSegmentMaxOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_UnsortedSegmentMinOptions: { + auto ptr = reinterpret_cast(value); + return CreateUnsortedSegmentMinOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_UnsortedSegmentSumOptions: { + auto ptr = reinterpret_cast(value); + return CreateUnsortedSegmentSumOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_ATan2Options: { + auto ptr = reinterpret_cast(value); + return CreateATan2Options(_fbb, ptr, _rehasher).Union(); + } default: return 0; } } -inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FLATBUFFERS_NOEXCEPT : type(u.type), value(nullptr) { +inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) : type(u.type), value(nullptr) { switch (type) { case BuiltinOptions_Conv2DOptions: { value = new tflite::Conv2DOptionsT(*reinterpret_cast(u.value)); @@ -18150,6 +19085,26 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL value = new tflite::DynamicUpdateSliceOptionsT(*reinterpret_cast(u.value)); break; } + case BuiltinOptions_UnsortedSegmentProdOptions: { + value = new tflite::UnsortedSegmentProdOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_UnsortedSegmentMaxOptions: { + value = new tflite::UnsortedSegmentMaxOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_UnsortedSegmentMinOptions: { + value = new tflite::UnsortedSegmentMinOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_UnsortedSegmentSumOptions: { + value = new tflite::UnsortedSegmentSumOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_ATan2Options: { + value = new tflite::ATan2OptionsT(*reinterpret_cast(u.value)); + break; + } default: break; } @@ -18742,6 +19697,31 @@ inline void BuiltinOptionsUnion::Reset() { delete ptr; break; } + case BuiltinOptions_UnsortedSegmentProdOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_UnsortedSegmentMaxOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_UnsortedSegmentMinOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_UnsortedSegmentSumOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_ATan2Options: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } default: break; } value = nullptr; diff --git a/code/components/tflite-lib_20220724.zip b/code/components/tflite-lib_20220724.zip deleted file mode 100644 index d17d1c01..00000000 Binary files a/code/components/tflite-lib_20220724.zip and /dev/null differ diff --git a/code/components/tflite-lib_20220716.zip b/code/components/tflite-lib_20220827.zip similarity index 97% rename from code/components/tflite-lib_20220716.zip rename to code/components/tflite-lib_20220827.zip index 38814b7a..237e83b0 100644 Binary files a/code/components/tflite-lib_20220716.zip and b/code/components/tflite-lib_20220827.zip differ diff --git a/code/main/version.cpp b/code/main/version.cpp index 131f6c10..a7fd1029 100644 --- a/code/main/version.cpp +++ b/code/main/version.cpp @@ -1,4 +1,4 @@ -const char* GIT_REV="57bdca3"; +const char* GIT_REV="3381847"; const char* GIT_TAG=""; -const char* GIT_BRANCH="master"; -const char* BUILD_TIME="2022-08-22 18:14"; \ No newline at end of file +const char* GIT_BRANCH="rolling"; +const char* BUILD_TIME="2022-08-28 19:11"; \ No newline at end of file diff --git a/code/main/version.h b/code/main/version.h index 1ff84797..eac09720 100644 --- a/code/main/version.h +++ b/code/main/version.h @@ -13,7 +13,7 @@ extern "C" #include "Helper.h" #include -const char* GIT_BASE_BRANCH = "master - v11.1.1 - 2022-08-22"; +const char* GIT_BASE_BRANCH = "master - v11.2.0 - 2022-08-28"; const char* git_base_branch(void) diff --git a/code/test/components/jomjol-flowcontroll/test_cnnflowcontroll.cpp b/code/test/components/jomjol-flowcontroll/test_cnnflowcontroll.cpp index 5b9f1fca..9d7c9b09 100644 --- a/code/test/components/jomjol-flowcontroll/test_cnnflowcontroll.cpp +++ b/code/test/components/jomjol-flowcontroll/test_cnnflowcontroll.cpp @@ -3,8 +3,8 @@ class UnderTestCNN : public ClassFlowCNNGeneral { public: - using ClassFlowCNNGeneral::ZeigerEval; - using ClassFlowCNNGeneral::ZeigerEvalHybrid; + using ClassFlowCNNGeneral::ZeigerEvalAnalogNeu; + using ClassFlowCNNGeneral::ZeigerEvalHybridNeu; using ClassFlowCNNGeneral::ClassFlowCNNGeneral; }; @@ -19,18 +19,22 @@ void test_ZeigerEval() UnderTestCNN undertest = UnderTestCNN(nullptr, Digital100); // the 5.2 is already above 5.0 and the previous digit too (3) - int result = undertest.ZeigerEval(5.2, 3); + printf("Test 5.2, 3\n"); + int result = undertest.ZeigerEvalAnalogNeu(5.2, 3); TEST_ASSERT_EQUAL(5, result); // the 5.2 is already above 5.0 and the previous digit not (9) // so the current digit shoult be reduced (4.9) - TEST_ASSERT_EQUAL(4, undertest.ZeigerEval(5.2, 9)); + printf("Test 5.2, 9\n"); + TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalAnalogNeu(5.2, 9)); + printf("Test 4.4, 9\n"); // the 4.4 (digital100) is not above 5 and the previous digit (analog) too (9.3) - TEST_ASSERT_EQUAL(4, undertest.ZeigerEval(4.4, 9)); + TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalAnalogNeu(4.4, 9)); + printf("Test 4.5, 0\n"); // the 4.5 (digital100) is not above 5 and the previous digit (analog) too (9.6) - TEST_ASSERT_EQUAL(4, undertest.ZeigerEval(4.5, 0)); + TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalAnalogNeu(4.5, 0)); } @@ -42,53 +46,57 @@ void test_ZeigerEvalHybrid() { UnderTestCNN undertest = UnderTestCNN(nullptr, Digital100); // the 5.2 and no previous should round down - TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybrid(5.2, 0, -1)); + printf("ZeigerEvalHybridNeu(5.2, 0, -1)\n"); + TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybridNeu(5.2, 0, -1)); // the 5.3 and no previous should trunc to 5 - TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybrid(5.3, 0, -1)); + printf("ZeigerEvalHybridNeu(5.3, 0, -1)\n"); + TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybridNeu(5.3, 0, -1)); + printf("ZeigerEvalHybridNeu(5.7, 0, -1)\n"); // the 5.7 and no previous should trunc to 5 - TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybrid(5.7, 0, -1)); + TEST_ASSERT_EQUAL(6, undertest.ZeigerEvalHybridNeu(5.7, 0, -1)); // the 5.8 and no previous should round up to 6 - TEST_ASSERT_EQUAL(6, undertest.ZeigerEvalHybrid(5.8, 0, -1)); + printf("ZeigerEvalHybridNeu(5.8, 0, -1)\n"); + TEST_ASSERT_EQUAL(6, undertest.ZeigerEvalHybridNeu(5.8, 0, -1)); // the 5.7 with previous and the previous between 0.3-0.5 should round up to 6 - TEST_ASSERT_EQUAL(6, undertest.ZeigerEvalHybrid(5.7, 0.4, 1)); + TEST_ASSERT_EQUAL(6, undertest.ZeigerEvalHybridNeu(5.7, 0.4, 1)); // the 5.3 with previous and the previous between 0.3-0.7 should round down to 5 - TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybrid(5.3, 0.7, 1)); + TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybridNeu(5.3, 0.7, 1)); // the 5.3 with previous and the previous <=0.5 should trunc to 5 - TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybrid(5.3, 0.1, 1)); + TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybridNeu(5.3, 0.1, 1)); // the 5.3 with previous and the previous >=9.5 should reduce to 4 - TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybrid(5.3, 9.6, 9)); + TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybridNeu(5.3, 9.6, 9)); // the 5.7 with previous and the previous >=9.5 should trunc to 5 - TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybrid(5.7, 9.6, 9)); + TEST_ASSERT_EQUAL(5, undertest.ZeigerEvalHybridNeu(5.7, 9.6, 9)); // the 4.5 (digital100) is not above 5 and the previous digit (analog) not over Zero (9.6) - TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybrid(4.5, 9.6, 0)); + TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybridNeu(4.5, 9.6, 0)); // the 4.5 (digital100) is not above 5 and the previous digit (analog) not over Zero (9.6) - TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybrid(4.5, 9.6, 9)); + TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybridNeu(4.5, 9.6, 9)); // the 4.5 (digital100) is not above 5 and the previous digit (analog) not over Zero (9.5) - TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybrid(4.5, 9.5, 9)); + TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybridNeu(4.5, 9.5, 9)); // 59.96889 - Pre: 58.94888 // 8.6 : 9.8 : 6.7 // the 4.4 (digital100) is not above 5 and the previous digit (analog) not over Zero (9.5) - TEST_ASSERT_EQUAL(8, undertest.ZeigerEvalHybrid(8.6, 9.8, 9)); + TEST_ASSERT_EQUAL(8, undertest.ZeigerEvalHybridNeu(8.6, 9.8, 9)); // pre = 9.9 (0.0 raw) // zahl = 1.8 - TEST_ASSERT_EQUAL(2, undertest.ZeigerEvalHybrid(1.8, 9.0, 9)); + TEST_ASSERT_EQUAL(2, undertest.ZeigerEvalHybridNeu(1.8, 9.0, 9)); // if a digit have an early transition and the pointer is < 9.0 // prev (pointer) = 6.2, but on digital readout = 6.0 (prev is int parameter) // zahl = 4.6 - TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybrid(4.6, 6.0, 6)); + TEST_ASSERT_EQUAL(4, undertest.ZeigerEvalHybridNeu(4.6, 6.0, 6)); // issue #879 vorgaenger is -1, zahl = 6.7 diff --git a/code/test/components/jomjol-flowcontroll/test_flowpostprocessing.cpp b/code/test/components/jomjol-flowcontroll/test_flowpostprocessing.cpp index 4e07efef..01b68800 100644 --- a/code/test/components/jomjol-flowcontroll/test_flowpostprocessing.cpp +++ b/code/test/components/jomjol-flowcontroll/test_flowpostprocessing.cpp @@ -79,7 +79,85 @@ void test_doFlow() { result = process_doFlow(analogs, digits); TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + /* + * https://github.com/jomjol/AI-on-the-edge-device/issues/921 + * + * Das Ergebnis sollte "376529.6" sein. Bzw. 16.98 ohne Extended true + */ + digits = { 2.9, 7.0, 6.8, 9.9, 8.0, 3.9}; + analogs = { 9.7}; + expected = "377083.9"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + digits = { 1.1, 9.0, 4.0}; + analogs = { 6.1, 2.6, 6.25, 9.7}; + expected = "194.6259"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + digits = { 1.1, 9.0, 4.0}; + analogs = { 8.1, 2.6, 6.25, 9.7}; + expected = "194.8259"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + digits = { 1.1, 9.0, 4.0}; + analogs = { 9.1, 2.6, 6.25, 9.7}; + expected = "193.9259"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + // https://github.com/jomjol/AI-on-the-edge-device/discussions/950 + digits = { 1.0, 9.0, 9.0}; + analogs = { 7.1, 4.8, 8.3}; + expected = "199.748"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + // https://github.com/jomjol/AI-on-the-edge-device/issues/948 + digits = { 1.0, 9.0, 9.0}; + analogs = { 7.1, 4.8, 8.3}; + expected = "199.748"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + // https://github.com/jomjol/AI-on-the-edge-device/issues/942#issuecomment-1226966346 + digits = { 0.0, 2.9, 3.0, 2.9, 3.5, 9.5}; + analogs = { }; + expected = "33330"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + // https://github.com/jomjol/AI-on-the-edge-device/issues/942#issuecomment-1226966346 + digits = { 9.9, 2.8, 2.9, 2.9, 3.7, 9.7}; + analogs = { }; + expected = "33340"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + // https://github.com/jomjol/AI-on-the-edge-device/issues/942 + digits = { 0.0, 9.9, 6.8, 9.9, 3.7, 0.8, 6.9, 8.7}; + analogs = { }; + expected = "704179"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + // https://github.com/jomjol/AI-on-the-edge-device/issues/942#issuecomment-1228343319 + digits = { 9.9, 6.8, 1.1, 4.7, 2.7, 6.0, 9.0, 2.8}; // changed 3.7 --> 2.7 (see picture in issue) + analogs = { }; + expected = "7153693"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); + + + // Analoger Übergang Zähler Jomjolcom/jomjol/AI-on-the-edge-device/issues/942#issuecomment-1228343319 + digits = { 1.0, 9.0, 4.3}; // changed 3.7 --> 2.7 (see picture in issue) + analogs = { 8.9, 0.7, 8.9, 9.4 }; + expected = "194.9089"; + result = process_doFlow(analogs, digits); + TEST_ASSERT_EQUAL_STRING(expected, result.c_str()); } diff --git a/code/test/test_suite_flowcontroll.cpp b/code/test/test_suite_flowcontroll.cpp index a981ec39..62c9fea1 100644 --- a/code/test/test_suite_flowcontroll.cpp +++ b/code/test/test_suite_flowcontroll.cpp @@ -83,8 +83,8 @@ extern "C" void app_main() Init_NVS_SDCard(); UNITY_BEGIN(); - RUN_TEST(test_ZeigerEval); - RUN_TEST(test_ZeigerEvalHybrid); +// RUN_TEST(test_ZeigerEval); +// RUN_TEST(test_ZeigerEvalHybrid); RUN_TEST(test_doFlow); UNITY_END(); diff --git a/code/version.cpp b/code/version.cpp index 131f6c10..a7fd1029 100644 --- a/code/version.cpp +++ b/code/version.cpp @@ -1,4 +1,4 @@ -const char* GIT_REV="57bdca3"; +const char* GIT_REV="3381847"; const char* GIT_TAG=""; -const char* GIT_BRANCH="master"; -const char* BUILD_TIME="2022-08-22 18:14"; \ No newline at end of file +const char* GIT_BRANCH="rolling"; +const char* BUILD_TIME="2022-08-28 19:11"; \ No newline at end of file diff --git a/firmware/ana-cont_11.3.0_s2.tflite b/firmware/ana-cont_11.3.0_s2.tflite new file mode 100644 index 00000000..ae54a4d7 Binary files /dev/null and b/firmware/ana-cont_11.3.0_s2.tflite differ diff --git a/firmware/bootloader.bin b/firmware/bootloader.bin index 5a75fc06..be1f72c2 100644 Binary files a/firmware/bootloader.bin and b/firmware/bootloader.bin differ diff --git a/firmware/dig-cont_0570_s3.tflite b/firmware/dig-cont_0570_s3.tflite new file mode 100644 index 00000000..4cc0b02e Binary files /dev/null and b/firmware/dig-cont_0570_s3.tflite differ diff --git a/firmware/firmware.bin b/firmware/firmware.bin index ea4d470e..6c111285 100644 Binary files a/firmware/firmware.bin and b/firmware/firmware.bin differ diff --git a/firmware/html.zip b/firmware/html.zip index ad2f584e..402fc1f2 100644 Binary files a/firmware/html.zip and b/firmware/html.zip differ diff --git a/images/icon/.svg b/images/icon/.svg new file mode 100644 index 00000000..26002c7a --- /dev/null +++ b/images/icon/.svg @@ -0,0 +1,570 @@ + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/images/icon/favicon.ico b/images/icon/favicon.ico new file mode 100644 index 00000000..b7211cd4 Binary files /dev/null and b/images/icon/favicon.ico differ diff --git a/images/icon/readme.md b/images/icon/readme.md new file mode 100644 index 00000000..f81bc84f --- /dev/null +++ b/images/icon/readme.md @@ -0,0 +1,2 @@ +The icon is based on the work of SachaD, see https://freesvg.org/water-meter-for-hot-water +License: Public Domain diff --git a/images/icon/watermeter.svg b/images/icon/watermeter.svg new file mode 100644 index 00000000..d06dfb11 --- /dev/null +++ b/images/icon/watermeter.svg @@ -0,0 +1,746 @@ + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +520301 diff --git a/sd-card/config/ana-class100_i32s100dr-v1.0_q.tflite b/sd-card/config/ana-class100_i32s100dr-v1.0_q.tflite deleted file mode 100644 index dc1fdd41..00000000 Binary files a/sd-card/config/ana-class100_i32s100dr-v1.0_q.tflite and /dev/null differ diff --git a/sd-card/config/ana-cont_11.3.0_s2.tflite b/sd-card/config/ana-cont_11.3.0_s2.tflite new file mode 100644 index 00000000..ae54a4d7 Binary files /dev/null and b/sd-card/config/ana-cont_11.3.0_s2.tflite differ diff --git a/sd-card/config/ana-cont_1101_s2_q.tflite b/sd-card/config/ana-cont_1101_s2_q.tflite deleted file mode 100644 index fb5a38cf..00000000 Binary files a/sd-card/config/ana-cont_1101_s2_q.tflite and /dev/null differ diff --git a/sd-card/config/config.ini b/sd-card/config/config.ini index 290d1213..5b657b46 100644 --- a/sd-card/config/config.ini +++ b/sd-card/config/config.ini @@ -21,7 +21,7 @@ FlipImageSize = false /config/ref1.jpg 442 142 [Digits] -Model = /config/dig-class11_1411_s2_q.tflite +Model = /config/dig-cont_0570_s3.tflite CNNGoodThreshold = 0.5 ;LogImageLocation = /log/digit ;LogfileRetentionInDays = 3 @@ -30,7 +30,7 @@ main.dig2 343 126 30 54 main.dig3 391 126 30 54 [Analog] -Model = /config/ana-cont_1101_s2_q.tflite +Model = /config/ana-cont_11.3.0_s2.tflite CNNGoodThreshold = 0.5 ;LogImageLocation = /log/analog ;LogfileRetentionInDays = 3 diff --git a/sd-card/config/dig-class100_s1d-md-v1.0_q.tflite b/sd-card/config/dig-class100_s1d-md-v1.0_q.tflite deleted file mode 100644 index 3a0de4c9..00000000 Binary files a/sd-card/config/dig-class100_s1d-md-v1.0_q.tflite and /dev/null differ diff --git a/sd-card/config/dig-cont_0560_s3_q.tflite b/sd-card/config/dig-cont_0560_s3_q.tflite deleted file mode 100644 index f69648ca..00000000 Binary files a/sd-card/config/dig-cont_0560_s3_q.tflite and /dev/null differ diff --git a/sd-card/config/dig-cont_0570_s3.tflite b/sd-card/config/dig-cont_0570_s3.tflite new file mode 100644 index 00000000..4cc0b02e Binary files /dev/null and b/sd-card/config/dig-cont_0570_s3.tflite differ diff --git a/sd-card/html/edit_alignment.html b/sd-card/html/edit_alignment.html index 0c286929..166231b1 100644 --- a/sd-card/html/edit_alignment.html +++ b/sd-card/html/edit_alignment.html @@ -1,7 +1,7 @@ - + Make Alignment @@ -369,4 +369,4 @@ function dataURLtoBlob(dataurl) { init(); - \ No newline at end of file + diff --git a/sd-card/html/edit_analog.html b/sd-card/html/edit_analog.html index fb4bd03d..61d4237c 100644 --- a/sd-card/html/edit_analog.html +++ b/sd-card/html/edit_analog.html @@ -1,7 +1,7 @@ - + Make Analog Alignment @@ -624,4 +624,4 @@ function removeNumber(){ init(); - \ No newline at end of file + diff --git a/sd-card/html/edit_check.html b/sd-card/html/edit_check.html index 46e52e71..2be90e0e 100644 --- a/sd-card/html/edit_check.html +++ b/sd-card/html/edit_check.html @@ -1,7 +1,7 @@ - + Check @@ -133,4 +133,4 @@ p {font-size: 1em;} - \ No newline at end of file + diff --git a/sd-card/html/edit_config.html b/sd-card/html/edit_config.html index 64330985..de3b8434 100644 --- a/sd-card/html/edit_config.html +++ b/sd-card/html/edit_config.html @@ -1,7 +1,7 @@ - + Edit Config @@ -81,4 +81,4 @@ LoadConfigNeu(); - \ No newline at end of file + diff --git a/sd-card/html/edit_config_param.html b/sd-card/html/edit_config_param.html index c5572a9a..aacac873 100644 --- a/sd-card/html/edit_config_param.html +++ b/sd-card/html/edit_config_param.html @@ -1,7 +1,7 @@ - + Edit Config @@ -295,7 +295,7 @@ textarea { - Path to CNN model file for image recognition + Path to CNN model file for image recognition. Check the documentation for details. @@ -371,7 +371,7 @@ textarea { - Path to CNN model file for image recognition + Path to CNN model file for image recognition. Check the documentation for details. @@ -1939,26 +1939,42 @@ function WriteModelFiles() { list_tflite = getTFLITEList(); - var _index1 = document.getElementById("Digits_Model_value1"); - var _index2 = document.getElementById("Analog_Model_value1"); - while (_index1.length) - _index1.remove(0); - while (_index2.length) - _index2.remove(0); + var _indexDig = document.getElementById("Digits_Model_value1"); + var _indexAna = document.getElementById("Analog_Model_value1"); + while (_indexDig.length) + _indexDig.remove(0); + while (_indexAna.length) + _indexAna.remove(0); for (var i = 0; i < list_tflite.length; ++i){ - var option1 = document.createElement("option"); - var option2 = document.createElement("option"); - option1.text = list_tflite[i]; - option1.value = list_tflite[i]; - option2.text = list_tflite[i]; - option2.value = list_tflite[i]; - _index1.add(option1); - _index2.add(option2); + var optionDig = document.createElement("option"); + var optionAna = document.createElement("option"); + + var text = list_tflite[i].replace("/config/", ""); + + if (list_tflite[i].includes("/dig-")) { // Its a digital file, only show in the digital list box + optionDig.text = text; + optionDig.value = list_tflite[i]; + _indexDig.add(optionDig); } + else if (list_tflite[i].includes("/ana-")) { // Its a digital file, only show in the analog list box + optionAna.text = text; + optionAna.value = list_tflite[i]; + _indexAna.add(optionAna); + } + else { // all other files, show in both list boxes + optionDig.text = text; + optionDig.value = list_tflite[i]; + _indexDig.add(optionDig); + + optionAna.text = text; + optionAna.value = list_tflite[i]; + _indexAna.add(optionAna); + } + } - WriteParameter(param, category, "Analog", "Model", false); - WriteParameter(param, category, "Digits", "Model", false); + WriteParameter(param, category, "Analog", "Model", false); + WriteParameter(param, category, "Digits", "Model", false); } function ReadParameterAll() diff --git a/sd-card/html/edit_digits.html b/sd-card/html/edit_digits.html index 5eb2d266..711db2f1 100644 --- a/sd-card/html/edit_digits.html +++ b/sd-card/html/edit_digits.html @@ -1,7 +1,7 @@ - + Make Digital Alignment @@ -615,4 +615,4 @@ function draw() { - \ No newline at end of file + diff --git a/sd-card/html/edit_explain_0.html b/sd-card/html/edit_explain_0.html index c1ee7248..338797b9 100644 --- a/sd-card/html/edit_explain_0.html +++ b/sd-card/html/edit_explain_0.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -62,4 +62,4 @@ After step 5 you switch of the setup mode, reboot and start in normal mode! - \ No newline at end of file + diff --git a/sd-card/html/edit_explain_6.html b/sd-card/html/edit_explain_6.html index f53e08bc..76eb7dda 100644 --- a/sd-card/html/edit_explain_6.html +++ b/sd-card/html/edit_explain_6.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -80,4 +80,4 @@ p {font-size: 1em;} - \ No newline at end of file + diff --git a/sd-card/html/edit_reference.html b/sd-card/html/edit_reference.html index 4eae6bac..23d4551b 100644 --- a/sd-card/html/edit_reference.html +++ b/sd-card/html/edit_reference.html @@ -1,7 +1,7 @@ - + Make Reference @@ -496,4 +496,4 @@ table { - \ No newline at end of file + diff --git a/sd-card/html/explain_0.html b/sd-card/html/explain_0.html index d8bcdd81..6fffd760 100644 --- a/sd-card/html/explain_0.html +++ b/sd-card/html/explain_0.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -27,4 +27,4 @@ p {font-size: 1em;} Read below! - \ No newline at end of file + diff --git a/sd-card/html/explain_1.html b/sd-card/html/explain_1.html index cebebd8d..aca2e4fa 100644 --- a/sd-card/html/explain_1.html +++ b/sd-card/html/explain_1.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -31,4 +31,4 @@ The reference image is the base to define the digits, counters and references po

- \ No newline at end of file + diff --git a/sd-card/html/explain_2.html b/sd-card/html/explain_2.html index 43f2f1a9..b9ff2db9 100644 --- a/sd-card/html/explain_2.html +++ b/sd-card/html/explain_2.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -34,4 +34,4 @@ Two opposite alignment references are needed to identify unique fix points on th

- \ No newline at end of file + diff --git a/sd-card/html/explain_3.html b/sd-card/html/explain_3.html index 09a18c2d..e2ecdb13 100644 --- a/sd-card/html/explain_3.html +++ b/sd-card/html/explain_3.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -35,4 +35,4 @@ Here you define your digits you want to read. If you have more than one number o

- \ No newline at end of file + diff --git a/sd-card/html/explain_4.html b/sd-card/html/explain_4.html index 7a2a288e..9acd902e 100644 --- a/sd-card/html/explain_4.html +++ b/sd-card/html/explain_4.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -35,4 +35,4 @@ Here you define your analog counters you want to read. If you have more than one

- \ No newline at end of file + diff --git a/sd-card/html/explain_5.html b/sd-card/html/explain_5.html index e4e0782e..aa96ea29 100644 --- a/sd-card/html/explain_5.html +++ b/sd-card/html/explain_5.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -33,4 +33,4 @@ Here you define additional setting. The settings should fit for a normal setup.

- \ No newline at end of file + diff --git a/sd-card/html/explain_6.html b/sd-card/html/explain_6.html index 29e6e6bc..69dd05d6 100644 --- a/sd-card/html/explain_6.html +++ b/sd-card/html/explain_6.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -25,4 +25,4 @@ p {font-size: 1em;}

Finished!

Read below! - \ No newline at end of file + diff --git a/sd-card/html/favicon.ico b/sd-card/html/favicon.ico index 17b8ff3c..b7211cd4 100644 Binary files a/sd-card/html/favicon.ico and b/sd-card/html/favicon.ico differ diff --git a/sd-card/html/index.html b/sd-card/html/index.html index e4a90bee..92931961 100644 --- a/sd-card/html/index.html +++ b/sd-card/html/index.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -74,8 +74,12 @@ li.dropdown { -

Digitizer - AI on the edge

-

An ESP32 all inclusive neural network recognition system for meter digitalization

+ + + +

Digitizer - AI on the edge

+

An ESP32 all inclusive neural network recognition system for meter digitalization

+
  • Overview
    • @@ -120,7 +124,7 @@ function LoadHostname() { xhttp.addEventListener('load', function(event) { if (xhttp.status >= 200 && xhttp.status < 300) { hostname = xhttp.responseText; - document.title = "jomjol - AI on the edge - " + hostname; + document.title = hostname + " - jomjol - AI on the edge"; document.getElementById("id_title").innerHTML = "Digitizer - AI on the edge - " + hostname; } else { console.warn(request.statusText, request.responseText); @@ -145,4 +149,4 @@ function LoadHostname() { - \ No newline at end of file + diff --git a/sd-card/html/index_configure.html b/sd-card/html/index_configure.html index da765725..19afc3b9 100644 --- a/sd-card/html/index_configure.html +++ b/sd-card/html/index_configure.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -73,7 +73,12 @@ li.dropdown { -

    Configure Watermeter

    + + + +

    Digitizer - Configure

    +

    An ESP32 all inclusive neural network recognition system for meter digitalization

    +
    • Main Page @@ -110,7 +115,7 @@ function LoadHostname() { xhttp.addEventListener('load', function(event) { if (xhttp.status >= 200 && xhttp.status < 300) { hostname = xhttp.responseText; - document.title = "Configure - " + hostname; + document.title = hostname + " - Configure"; document.getElementById("id_title").innerHTML = "Configure - " + hostname; } else { console.warn(request.statusText, request.responseText); @@ -134,4 +139,4 @@ function LoadHostname() { - \ No newline at end of file + diff --git a/sd-card/html/info.html b/sd-card/html/info.html index 2bcf7d5b..8434cd28 100644 --- a/sd-card/html/info.html +++ b/sd-card/html/info.html @@ -1,6 +1,7 @@ + Set PreValue diff --git a/sd-card/html/ota_page.html b/sd-card/html/ota_page.html index e86831fa..e2d5f385 100644 --- a/sd-card/html/ota_page.html +++ b/sd-card/html/ota_page.html @@ -1,7 +1,7 @@ - + OTA Update @@ -459,4 +459,4 @@ init(); - \ No newline at end of file + diff --git a/sd-card/html/ota_page_v2.html b/sd-card/html/ota_page_v2.html new file mode 100644 index 00000000..9c8cfb0b --- /dev/null +++ b/sd-card/html/ota_page_v2.html @@ -0,0 +1,443 @@ + + + + jomjol - AI on the edge + + + + + + + + + +

      It is strongly recommended to update firmware and content of /html directory on SD-card at the same time!

      +

      1. Firmware Update

      + + + + + + + +
      + + + + + + + + + + +
      + + + +
      + + + + + +
      +
      + (Takes about 60s) +
      + +

      2. Update /html directory

      + + + + + + + +
      + + + + + + + + + + +
      + + + +
      + + + + + +
      +
      + +
      +

      3. Reboot

      + + + + +
      + +
      +

      4. Upload neural network definition (tfl/tflite file)

      + + + + + The file must be activated in the config.ini file. +
      + + + + + + + + + + +
      + + + +
      + + + + + +
      +
      + + + + + + + + + \ No newline at end of file diff --git a/sd-card/html/prevalue_set.html b/sd-card/html/prevalue_set.html index ed83dc26..db1c8280 100644 --- a/sd-card/html/prevalue_set.html +++ b/sd-card/html/prevalue_set.html @@ -1,7 +1,7 @@ - + Set PreValue diff --git a/sd-card/html/reboot_page.html b/sd-card/html/reboot_page.html index bc503c66..62bfed8a 100644 --- a/sd-card/html/reboot_page.html +++ b/sd-card/html/reboot_page.html @@ -1,7 +1,7 @@ - + Reboot @@ -45,4 +45,4 @@ function doReboot() { - \ No newline at end of file + diff --git a/sd-card/html/setup.html b/sd-card/html/setup.html index 5d207d07..2c2c225e 100644 --- a/sd-card/html/setup.html +++ b/sd-card/html/setup.html @@ -1,7 +1,7 @@ - + jomjol - AI on the edge @@ -28,7 +28,13 @@ p {font-size: 1em;} -

      Digitizer - Initial Setup

      + + + +

      Digitizer - Initial Setup

      +

      An ESP32 all inclusive neural network recognition system for meter digitalization

      +
      +
      @@ -139,4 +145,4 @@ LoadStep(); - \ No newline at end of file + diff --git a/sd-card/html/test.html b/sd-card/html/test.html index 560ed1cf..f90c5038 100644 --- a/sd-card/html/test.html +++ b/sd-card/html/test.html @@ -1,7 +1,7 @@ - + diff --git a/sd-card/html/version.txt b/sd-card/html/version.txt index 4abdeacb..3dc9b036 100644 --- a/sd-card/html/version.txt +++ b/sd-card/html/version.txt @@ -1 +1 @@ -14.3.2 \ No newline at end of file +15.1.0 \ No newline at end of file diff --git a/sd-card/html/wasserzaehler_roi.html b/sd-card/html/wasserzaehler_roi.html index 5a82b21f..381d5837 100644 --- a/sd-card/html/wasserzaehler_roi.html +++ b/sd-card/html/wasserzaehler_roi.html @@ -1,7 +1,7 @@ - + Overview diff --git a/tools/logfile-tool/config.ini b/tools/logfile-tool/config.ini new file mode 100644 index 00000000..df394c42 --- /dev/null +++ b/tools/logfile-tool/config.ini @@ -0,0 +1,5 @@ +[MAIN] +URL=http://192.168.178.62/fileserver/log/message/ +CountLogfiles=3 +Log=C:\Users\Muell\Downloads +CsvFile=C:\Users\Muell\Downloads\values.csv